2024-11-11 20:52:37,277 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-11 20:52:37,288 main DEBUG Took 0.009180 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-11 20:52:37,289 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-11 20:52:37,289 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-11 20:52:37,290 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-11 20:52:37,291 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:52:37,298 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-11 20:52:37,310 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:52:37,311 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:52:37,312 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:52:37,313 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:52:37,313 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:52:37,313 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:52:37,314 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:52:37,315 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:52:37,315 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:52:37,315 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:52:37,316 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:52:37,317 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:52:37,317 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:52:37,317 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:52:37,318 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:52:37,318 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:52:37,319 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:52:37,319 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:52:37,319 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:52:37,320 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:52:37,320 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:52:37,320 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:52:37,321 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:52:37,321 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 20:52:37,321 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:52:37,322 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-11 20:52:37,323 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 20:52:37,324 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-11 20:52:37,326 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-11 20:52:37,326 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-11 20:52:37,327 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-11 20:52:37,328 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-11 20:52:37,336 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-11 20:52:37,338 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-11 20:52:37,340 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-11 20:52:37,340 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-11 20:52:37,341 main DEBUG createAppenders(={Console}) 2024-11-11 20:52:37,342 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-11 20:52:37,342 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-11 20:52:37,342 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-11 20:52:37,343 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-11 20:52:37,343 main DEBUG OutputStream closed 2024-11-11 20:52:37,343 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-11 20:52:37,344 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-11 20:52:37,344 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-11 20:52:37,413 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-11 20:52:37,415 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-11 20:52:37,417 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-11 20:52:37,418 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-11 20:52:37,418 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-11 20:52:37,419 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-11 20:52:37,419 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-11 20:52:37,419 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-11 20:52:37,420 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-11 20:52:37,420 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-11 20:52:37,420 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-11 20:52:37,421 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-11 20:52:37,421 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-11 20:52:37,421 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-11 20:52:37,422 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-11 20:52:37,422 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-11 20:52:37,422 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-11 20:52:37,423 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-11 20:52:37,425 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11 20:52:37,426 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-11 20:52:37,426 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-11 20:52:37,427 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-11T20:52:37,711 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925 2024-11-11 20:52:37,714 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-11 20:52:37,714 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11T20:52:37,725 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-11T20:52:37,756 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-11T20:52:37,772 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/cluster_aeef4fcb-2790-f92a-34f3-fa13c2a932a9, deleteOnExit=true 2024-11-11T20:52:37,772 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-11T20:52:37,773 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/test.cache.data in system properties and HBase conf 2024-11-11T20:52:37,774 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/hadoop.tmp.dir in system properties and HBase conf 2024-11-11T20:52:37,774 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/hadoop.log.dir in system properties and HBase conf 2024-11-11T20:52:37,775 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-11T20:52:37,775 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-11T20:52:37,776 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-11T20:52:37,853 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-11T20:52:38,016 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T20:52:38,028 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-11T20:52:38,029 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-11T20:52:38,032 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-11T20:52:38,033 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T20:52:38,033 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-11T20:52:38,034 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-11T20:52:38,035 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T20:52:38,035 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T20:52:38,036 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-11T20:52:38,037 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/nfs.dump.dir in system properties and HBase conf 2024-11-11T20:52:38,037 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/java.io.tmpdir in system properties and HBase conf 2024-11-11T20:52:38,038 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T20:52:38,039 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-11T20:52:38,039 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-11T20:52:39,099 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-11T20:52:39,197 INFO [Time-limited test {}] log.Log(170): Logging initialized @2632ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-11T20:52:39,277 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T20:52:39,339 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T20:52:39,368 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T20:52:39,368 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T20:52:39,369 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T20:52:39,381 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T20:52:39,384 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/hadoop.log.dir/,AVAILABLE} 2024-11-11T20:52:39,385 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T20:52:39,568 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/java.io.tmpdir/jetty-localhost-46573-hadoop-hdfs-3_4_1-tests_jar-_-any-8864234853078123237/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T20:52:39,575 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:46573} 2024-11-11T20:52:39,576 INFO [Time-limited test {}] server.Server(415): Started @3012ms 2024-11-11T20:52:39,912 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T20:52:39,919 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T20:52:39,920 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T20:52:39,920 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T20:52:39,921 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T20:52:39,922 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/hadoop.log.dir/,AVAILABLE} 2024-11-11T20:52:39,922 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T20:52:40,030 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/java.io.tmpdir/jetty-localhost-33261-hadoop-hdfs-3_4_1-tests_jar-_-any-10596440661867403313/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T20:52:40,031 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:33261} 2024-11-11T20:52:40,031 INFO [Time-limited test {}] server.Server(415): Started @3467ms 2024-11-11T20:52:40,084 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T20:52:40,482 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/cluster_aeef4fcb-2790-f92a-34f3-fa13c2a932a9/data/data2/current/BP-1400522365-172.17.0.2-1731358358860/current, will proceed with Du for space computation calculation, 2024-11-11T20:52:40,482 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/cluster_aeef4fcb-2790-f92a-34f3-fa13c2a932a9/data/data1/current/BP-1400522365-172.17.0.2-1731358358860/current, will proceed with Du for space computation calculation, 2024-11-11T20:52:40,523 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T20:52:40,580 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925 2024-11-11T20:52:40,587 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf2b6ad2aec8f6c3b with lease ID 0x89b3b844fd4d47e5: Processing first storage report for DS-d472e840-ebf6-481e-9bfc-2877b94b2b85 from datanode DatanodeRegistration(127.0.0.1:34289, datanodeUuid=4d7c653e-098e-43e9-9ddd-92046dcfd7fe, infoPort=33601, infoSecurePort=0, ipcPort=46775, storageInfo=lv=-57;cid=testClusterID;nsid=2111173571;c=1731358358860) 2024-11-11T20:52:40,589 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf2b6ad2aec8f6c3b with lease ID 0x89b3b844fd4d47e5: from storage DS-d472e840-ebf6-481e-9bfc-2877b94b2b85 node DatanodeRegistration(127.0.0.1:34289, datanodeUuid=4d7c653e-098e-43e9-9ddd-92046dcfd7fe, infoPort=33601, infoSecurePort=0, ipcPort=46775, storageInfo=lv=-57;cid=testClusterID;nsid=2111173571;c=1731358358860), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-11T20:52:40,589 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf2b6ad2aec8f6c3b with lease ID 0x89b3b844fd4d47e5: Processing first storage report for DS-e3bcf139-9db7-4457-a79f-afb5afeb4704 from datanode DatanodeRegistration(127.0.0.1:34289, datanodeUuid=4d7c653e-098e-43e9-9ddd-92046dcfd7fe, infoPort=33601, infoSecurePort=0, ipcPort=46775, storageInfo=lv=-57;cid=testClusterID;nsid=2111173571;c=1731358358860) 2024-11-11T20:52:40,590 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf2b6ad2aec8f6c3b with lease ID 0x89b3b844fd4d47e5: from storage DS-e3bcf139-9db7-4457-a79f-afb5afeb4704 node DatanodeRegistration(127.0.0.1:34289, datanodeUuid=4d7c653e-098e-43e9-9ddd-92046dcfd7fe, infoPort=33601, infoSecurePort=0, ipcPort=46775, storageInfo=lv=-57;cid=testClusterID;nsid=2111173571;c=1731358358860), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T20:52:40,661 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/cluster_aeef4fcb-2790-f92a-34f3-fa13c2a932a9/zookeeper_0, clientPort=61333, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/cluster_aeef4fcb-2790-f92a-34f3-fa13c2a932a9/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/cluster_aeef4fcb-2790-f92a-34f3-fa13c2a932a9/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-11T20:52:40,672 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=61333 2024-11-11T20:52:40,686 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T20:52:40,689 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T20:52:40,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741825_1001 (size=7) 2024-11-11T20:52:41,338 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23 with version=8 2024-11-11T20:52:41,339 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/hbase-staging 2024-11-11T20:52:41,417 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-11T20:52:41,617 INFO [Time-limited test {}] client.ConnectionUtils(128): master/51ca66f7ee3c:0 server-side Connection retries=45 2024-11-11T20:52:41,627 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:52:41,628 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T20:52:41,632 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T20:52:41,632 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:52:41,632 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T20:52:41,753 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-11T20:52:41,802 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-11T20:52:41,810 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-11T20:52:41,813 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T20:52:41,833 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 41692 (auto-detected) 2024-11-11T20:52:41,834 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-11T20:52:41,849 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38429 2024-11-11T20:52:41,868 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:38429 connecting to ZooKeeper ensemble=127.0.0.1:61333 2024-11-11T20:52:41,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:384290x0, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T20:52:41,897 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38429-0x100308f60b00000 connected 2024-11-11T20:52:41,923 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T20:52:41,926 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T20:52:41,937 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T20:52:41,941 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23, hbase.cluster.distributed=false 2024-11-11T20:52:41,963 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T20:52:41,968 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38429 2024-11-11T20:52:41,969 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38429 2024-11-11T20:52:41,973 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38429 2024-11-11T20:52:41,976 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38429 2024-11-11T20:52:41,978 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38429 2024-11-11T20:52:42,100 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/51ca66f7ee3c:0 server-side Connection retries=45 2024-11-11T20:52:42,102 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:52:42,102 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T20:52:42,103 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T20:52:42,103 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T20:52:42,103 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T20:52:42,106 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T20:52:42,109 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T20:52:42,110 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:44669 2024-11-11T20:52:42,112 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44669 connecting to ZooKeeper ensemble=127.0.0.1:61333 2024-11-11T20:52:42,113 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T20:52:42,116 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T20:52:42,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:446690x0, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T20:52:42,121 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:446690x0, quorum=127.0.0.1:61333, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T20:52:42,122 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44669-0x100308f60b00001 connected 2024-11-11T20:52:42,125 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T20:52:42,132 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T20:52:42,135 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44669-0x100308f60b00001, quorum=127.0.0.1:61333, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T20:52:42,141 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44669-0x100308f60b00001, quorum=127.0.0.1:61333, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T20:52:42,142 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44669 2024-11-11T20:52:42,142 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44669 2024-11-11T20:52:42,142 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44669 2024-11-11T20:52:42,143 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44669 2024-11-11T20:52:42,143 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44669 2024-11-11T20:52:42,157 DEBUG [M:0;51ca66f7ee3c:38429 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;51ca66f7ee3c:38429 2024-11-11T20:52:42,158 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/51ca66f7ee3c,38429,1731358361462 2024-11-11T20:52:42,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44669-0x100308f60b00001, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T20:52:42,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T20:52:42,166 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/51ca66f7ee3c,38429,1731358361462 2024-11-11T20:52:42,185 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44669-0x100308f60b00001, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T20:52:42,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:42,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44669-0x100308f60b00001, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:42,187 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T20:52:42,188 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/51ca66f7ee3c,38429,1731358361462 from backup master directory 2024-11-11T20:52:42,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44669-0x100308f60b00001, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T20:52:42,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/51ca66f7ee3c,38429,1731358361462 2024-11-11T20:52:42,190 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T20:52:42,191 WARN [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T20:52:42,191 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=51ca66f7ee3c,38429,1731358361462 2024-11-11T20:52:42,193 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-11T20:52:42,194 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-11T20:52:42,248 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/hbase.id] with ID: e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8 2024-11-11T20:52:42,248 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/.tmp/hbase.id 2024-11-11T20:52:42,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741826_1002 (size=42) 2024-11-11T20:52:42,661 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/.tmp/hbase.id]:[hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/hbase.id] 2024-11-11T20:52:42,762 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T20:52:42,774 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-11T20:52:42,891 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 101ms. 2024-11-11T20:52:42,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:42,911 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44669-0x100308f60b00001, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:43,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741827_1003 (size=196) 2024-11-11T20:52:43,470 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T20:52:43,510 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-11T20:52:43,563 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:43,571 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T20:52:43,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741828_1004 (size=1189) 2024-11-11T20:52:43,716 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/MasterData/data/master/store 2024-11-11T20:52:43,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741829_1005 (size=34) 2024-11-11T20:52:44,059 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-11T20:52:44,128 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:52:44,129 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T20:52:44,129 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:52:44,130 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:52:44,196 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T20:52:44,196 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:52:44,196 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:52:44,198 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731358364129Disabling compacts and flushes for region at 1731358364129Disabling writes for close at 1731358364196 (+67 ms)Writing region close event to WAL at 1731358364196Closed at 1731358364196 2024-11-11T20:52:44,247 WARN [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/MasterData/data/master/store/.initializing 2024-11-11T20:52:44,248 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/MasterData/WALs/51ca66f7ee3c,38429,1731358361462 2024-11-11T20:52:44,257 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T20:52:44,272 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=51ca66f7ee3c%2C38429%2C1731358361462, suffix=, logDir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/MasterData/WALs/51ca66f7ee3c,38429,1731358361462, archiveDir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/MasterData/oldWALs, maxLogs=10 2024-11-11T20:52:44,293 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/MasterData/WALs/51ca66f7ee3c,38429,1731358361462/51ca66f7ee3c%2C38429%2C1731358361462.1731358364276, exclude list is [], retry=0 2024-11-11T20:52:44,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34289,DS-d472e840-ebf6-481e-9bfc-2877b94b2b85,DISK] 2024-11-11T20:52:44,313 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-11T20:52:44,348 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/MasterData/WALs/51ca66f7ee3c,38429,1731358361462/51ca66f7ee3c%2C38429%2C1731358361462.1731358364276 2024-11-11T20:52:44,349 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33601:33601)] 2024-11-11T20:52:44,350 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:52:44,350 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:52:44,354 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:44,355 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:44,388 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:44,411 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-11T20:52:44,415 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:44,417 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:44,417 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:44,420 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-11T20:52:44,421 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:44,422 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:52:44,422 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:44,425 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-11T20:52:44,425 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:44,426 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:52:44,426 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:44,429 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-11T20:52:44,430 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:44,431 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:52:44,431 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:44,435 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:44,436 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:44,442 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:44,443 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:44,447 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T20:52:44,450 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T20:52:44,455 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:52:44,457 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67684570, jitterRate=0.008578687906265259}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T20:52:44,464 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731358364366Initializing all the Stores at 1731358364368 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358364368Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358364369 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358364370 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358364370Cleaning up temporary data from old regions at 1731358364443 (+73 ms)Region opened successfully at 1731358364464 (+21 ms) 2024-11-11T20:52:44,466 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-11T20:52:44,503 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3559a3a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=51ca66f7ee3c/172.17.0.2:0 2024-11-11T20:52:44,541 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-11T20:52:44,555 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-11T20:52:44,555 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-11T20:52:44,559 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T20:52:44,560 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-11T20:52:44,565 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-11T20:52:44,565 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-11T20:52:44,593 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-11T20:52:44,602 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-11T20:52:44,604 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-11T20:52:44,607 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-11T20:52:44,609 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-11T20:52:44,610 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-11T20:52:44,613 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-11T20:52:44,616 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-11T20:52:44,618 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-11T20:52:44,619 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-11T20:52:44,620 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-11T20:52:44,636 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-11T20:52:44,637 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-11T20:52:44,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44669-0x100308f60b00001, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T20:52:44,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T20:52:44,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44669-0x100308f60b00001, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:44,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:44,643 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=51ca66f7ee3c,38429,1731358361462, sessionid=0x100308f60b00000, setting cluster-up flag (Was=false) 2024-11-11T20:52:44,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44669-0x100308f60b00001, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:44,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:44,656 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-11T20:52:44,658 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=51ca66f7ee3c,38429,1731358361462 2024-11-11T20:52:44,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:44,662 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44669-0x100308f60b00001, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:44,667 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-11T20:52:44,669 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=51ca66f7ee3c,38429,1731358361462 2024-11-11T20:52:44,674 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-11T20:52:44,740 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T20:52:44,749 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-11T20:52:44,755 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-11T20:52:44,756 INFO [RS:0;51ca66f7ee3c:44669 {}] regionserver.HRegionServer(746): ClusterId : e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8 2024-11-11T20:52:44,758 DEBUG [RS:0;51ca66f7ee3c:44669 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T20:52:44,763 DEBUG [RS:0;51ca66f7ee3c:44669 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T20:52:44,760 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 51ca66f7ee3c,38429,1731358361462 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-11T20:52:44,763 DEBUG [RS:0;51ca66f7ee3c:44669 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T20:52:44,766 DEBUG [RS:0;51ca66f7ee3c:44669 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T20:52:44,767 DEBUG [RS:0;51ca66f7ee3c:44669 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30a3edb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=51ca66f7ee3c/172.17.0.2:0 2024-11-11T20:52:44,768 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/51ca66f7ee3c:0, corePoolSize=5, maxPoolSize=5 2024-11-11T20:52:44,768 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/51ca66f7ee3c:0, corePoolSize=5, maxPoolSize=5 2024-11-11T20:52:44,768 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/51ca66f7ee3c:0, corePoolSize=5, maxPoolSize=5 2024-11-11T20:52:44,769 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/51ca66f7ee3c:0, corePoolSize=5, maxPoolSize=5 2024-11-11T20:52:44,769 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/51ca66f7ee3c:0, corePoolSize=10, maxPoolSize=10 2024-11-11T20:52:44,769 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:44,769 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/51ca66f7ee3c:0, corePoolSize=2, maxPoolSize=2 2024-11-11T20:52:44,770 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:44,775 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731358394775 2024-11-11T20:52:44,777 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-11T20:52:44,778 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T20:52:44,778 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-11T20:52:44,778 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-11T20:52:44,781 DEBUG [RS:0;51ca66f7ee3c:44669 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;51ca66f7ee3c:44669 2024-11-11T20:52:44,782 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-11T20:52:44,782 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-11T20:52:44,782 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-11T20:52:44,782 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-11T20:52:44,784 INFO [RS:0;51ca66f7ee3c:44669 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T20:52:44,784 INFO [RS:0;51ca66f7ee3c:44669 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T20:52:44,784 DEBUG [RS:0;51ca66f7ee3c:44669 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T20:52:44,783 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:44,786 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:44,786 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-11T20:52:44,787 INFO [RS:0;51ca66f7ee3c:44669 {}] regionserver.HRegionServer(2659): reportForDuty to master=51ca66f7ee3c,38429,1731358361462 with port=44669, startcode=1731358362066 2024-11-11T20:52:44,788 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-11T20:52:44,789 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-11T20:52:44,790 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-11T20:52:44,793 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-11T20:52:44,793 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-11T20:52:44,797 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.large.0-1731358364794,5,FailOnTimeoutGroup] 2024-11-11T20:52:44,798 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.small.0-1731358364797,5,FailOnTimeoutGroup] 2024-11-11T20:52:44,798 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:44,798 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-11T20:52:44,800 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:44,800 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:44,800 DEBUG [RS:0;51ca66f7ee3c:44669 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T20:52:44,803 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741831_1007 (size=1321) 2024-11-11T20:52:44,805 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-11T20:52:44,806 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23 2024-11-11T20:52:44,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741832_1008 (size=32) 2024-11-11T20:52:44,847 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34685, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T20:52:44,853 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38429 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:44,855 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38429 {}] master.ServerManager(517): Registering regionserver=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:44,869 DEBUG [RS:0;51ca66f7ee3c:44669 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23 2024-11-11T20:52:44,869 DEBUG [RS:0;51ca66f7ee3c:44669 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:45199 2024-11-11T20:52:44,869 DEBUG [RS:0;51ca66f7ee3c:44669 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T20:52:44,873 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T20:52:44,874 DEBUG [RS:0;51ca66f7ee3c:44669 {}] zookeeper.ZKUtil(111): regionserver:44669-0x100308f60b00001, quorum=127.0.0.1:61333, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:44,874 WARN [RS:0;51ca66f7ee3c:44669 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T20:52:44,874 INFO [RS:0;51ca66f7ee3c:44669 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T20:52:44,874 DEBUG [RS:0;51ca66f7ee3c:44669 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/WALs/51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:44,877 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [51ca66f7ee3c,44669,1731358362066] 2024-11-11T20:52:44,898 INFO [RS:0;51ca66f7ee3c:44669 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T20:52:44,913 INFO [RS:0;51ca66f7ee3c:44669 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T20:52:44,917 INFO [RS:0;51ca66f7ee3c:44669 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T20:52:44,918 INFO [RS:0;51ca66f7ee3c:44669 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:44,918 INFO [RS:0;51ca66f7ee3c:44669 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T20:52:44,923 INFO [RS:0;51ca66f7ee3c:44669 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T20:52:44,924 INFO [RS:0;51ca66f7ee3c:44669 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:44,925 DEBUG [RS:0;51ca66f7ee3c:44669 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:44,925 DEBUG [RS:0;51ca66f7ee3c:44669 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:44,925 DEBUG [RS:0;51ca66f7ee3c:44669 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:44,925 DEBUG [RS:0;51ca66f7ee3c:44669 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:44,925 DEBUG [RS:0;51ca66f7ee3c:44669 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:44,926 DEBUG [RS:0;51ca66f7ee3c:44669 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/51ca66f7ee3c:0, corePoolSize=2, maxPoolSize=2 2024-11-11T20:52:44,926 DEBUG [RS:0;51ca66f7ee3c:44669 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:44,926 DEBUG [RS:0;51ca66f7ee3c:44669 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:44,926 DEBUG [RS:0;51ca66f7ee3c:44669 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:44,926 DEBUG [RS:0;51ca66f7ee3c:44669 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:44,926 DEBUG [RS:0;51ca66f7ee3c:44669 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:44,926 DEBUG [RS:0;51ca66f7ee3c:44669 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/51ca66f7ee3c:0, corePoolSize=1, maxPoolSize=1 2024-11-11T20:52:44,927 DEBUG [RS:0;51ca66f7ee3c:44669 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/51ca66f7ee3c:0, corePoolSize=3, maxPoolSize=3 2024-11-11T20:52:44,927 DEBUG [RS:0;51ca66f7ee3c:44669 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0, corePoolSize=3, maxPoolSize=3 2024-11-11T20:52:44,929 INFO [RS:0;51ca66f7ee3c:44669 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:44,929 INFO [RS:0;51ca66f7ee3c:44669 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:44,929 INFO [RS:0;51ca66f7ee3c:44669 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:44,929 INFO [RS:0;51ca66f7ee3c:44669 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:44,929 INFO [RS:0;51ca66f7ee3c:44669 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:44,930 INFO [RS:0;51ca66f7ee3c:44669 {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,44669,1731358362066-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T20:52:44,947 INFO [RS:0;51ca66f7ee3c:44669 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T20:52:44,949 INFO [RS:0;51ca66f7ee3c:44669 {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,44669,1731358362066-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:44,950 INFO [RS:0;51ca66f7ee3c:44669 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:44,950 INFO [RS:0;51ca66f7ee3c:44669 {}] regionserver.Replication(171): 51ca66f7ee3c,44669,1731358362066 started 2024-11-11T20:52:44,976 INFO [RS:0;51ca66f7ee3c:44669 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:44,977 INFO [RS:0;51ca66f7ee3c:44669 {}] regionserver.HRegionServer(1482): Serving as 51ca66f7ee3c,44669,1731358362066, RpcServer on 51ca66f7ee3c/172.17.0.2:44669, sessionid=0x100308f60b00001 2024-11-11T20:52:44,978 DEBUG [RS:0;51ca66f7ee3c:44669 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T20:52:44,978 DEBUG [RS:0;51ca66f7ee3c:44669 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:44,978 DEBUG [RS:0;51ca66f7ee3c:44669 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '51ca66f7ee3c,44669,1731358362066' 2024-11-11T20:52:44,978 DEBUG [RS:0;51ca66f7ee3c:44669 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T20:52:44,980 DEBUG [RS:0;51ca66f7ee3c:44669 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T20:52:44,980 DEBUG [RS:0;51ca66f7ee3c:44669 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T20:52:44,980 DEBUG [RS:0;51ca66f7ee3c:44669 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T20:52:44,981 DEBUG [RS:0;51ca66f7ee3c:44669 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:44,981 DEBUG [RS:0;51ca66f7ee3c:44669 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '51ca66f7ee3c,44669,1731358362066' 2024-11-11T20:52:44,981 DEBUG [RS:0;51ca66f7ee3c:44669 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T20:52:44,982 DEBUG [RS:0;51ca66f7ee3c:44669 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T20:52:44,982 DEBUG [RS:0;51ca66f7ee3c:44669 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T20:52:44,982 INFO [RS:0;51ca66f7ee3c:44669 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T20:52:44,982 INFO [RS:0;51ca66f7ee3c:44669 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T20:52:45,090 INFO [RS:0;51ca66f7ee3c:44669 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T20:52:45,096 INFO [RS:0;51ca66f7ee3c:44669 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=51ca66f7ee3c%2C44669%2C1731358362066, suffix=, logDir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/WALs/51ca66f7ee3c,44669,1731358362066, archiveDir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/oldWALs, maxLogs=32 2024-11-11T20:52:45,118 DEBUG [RS:0;51ca66f7ee3c:44669 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/WALs/51ca66f7ee3c,44669,1731358362066/51ca66f7ee3c%2C44669%2C1731358362066.1731358365099, exclude list is [], retry=0 2024-11-11T20:52:45,126 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34289,DS-d472e840-ebf6-481e-9bfc-2877b94b2b85,DISK] 2024-11-11T20:52:45,130 INFO [RS:0;51ca66f7ee3c:44669 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/WALs/51ca66f7ee3c,44669,1731358362066/51ca66f7ee3c%2C44669%2C1731358362066.1731358365099 2024-11-11T20:52:45,130 DEBUG [RS:0;51ca66f7ee3c:44669 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33601:33601)] 2024-11-11T20:52:45,225 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:52:45,229 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T20:52:45,234 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T20:52:45,235 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:45,237 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:45,237 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T20:52:45,241 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T20:52:45,242 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:45,243 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:45,243 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T20:52:45,246 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T20:52:45,246 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:45,247 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:45,248 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T20:52:45,250 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T20:52:45,251 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:45,252 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:45,252 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T20:52:45,254 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/hbase/meta/1588230740 2024-11-11T20:52:45,255 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/hbase/meta/1588230740 2024-11-11T20:52:45,258 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T20:52:45,258 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T20:52:45,259 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T20:52:45,261 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T20:52:45,265 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:52:45,266 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60910090, jitterRate=-0.09236893057823181}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T20:52:45,268 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731358365225Initializing all the Stores at 1731358365229 (+4 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358365229Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358365229Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358365229Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358365229Cleaning up temporary data from old regions at 1731358365258 (+29 ms)Region opened successfully at 1731358365268 (+10 ms) 2024-11-11T20:52:45,268 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T20:52:45,268 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T20:52:45,268 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T20:52:45,269 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T20:52:45,269 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T20:52:45,270 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T20:52:45,271 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731358365268Disabling compacts and flushes for region at 1731358365268Disabling writes for close at 1731358365269 (+1 ms)Writing region close event to WAL at 1731358365270 (+1 ms)Closed at 1731358365270 2024-11-11T20:52:45,275 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T20:52:45,275 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-11T20:52:45,282 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T20:52:45,291 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:52:45,294 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-11T20:52:45,448 DEBUG [51ca66f7ee3c:38429 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-11T20:52:45,463 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:45,471 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,44669,1731358362066, state=OPENING 2024-11-11T20:52:45,476 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-11T20:52:45,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:45,477 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44669-0x100308f60b00001, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:52:45,478 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:52:45,478 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:52:45,479 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T20:52:45,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,44669,1731358362066}] 2024-11-11T20:52:45,656 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T20:52:45,659 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35577, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T20:52:45,670 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-11T20:52:45,671 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T20:52:45,671 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T20:52:45,674 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=51ca66f7ee3c%2C44669%2C1731358362066.meta, suffix=.meta, logDir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/WALs/51ca66f7ee3c,44669,1731358362066, archiveDir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/oldWALs, maxLogs=32 2024-11-11T20:52:45,688 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/WALs/51ca66f7ee3c,44669,1731358362066/51ca66f7ee3c%2C44669%2C1731358362066.meta.1731358365676.meta, exclude list is [], retry=0 2024-11-11T20:52:45,692 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34289,DS-d472e840-ebf6-481e-9bfc-2877b94b2b85,DISK] 2024-11-11T20:52:45,695 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/WALs/51ca66f7ee3c,44669,1731358362066/51ca66f7ee3c%2C44669%2C1731358362066.meta.1731358365676.meta 2024-11-11T20:52:45,696 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:33601:33601)] 2024-11-11T20:52:45,696 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:52:45,697 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T20:52:45,700 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T20:52:45,704 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T20:52:45,708 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T20:52:45,708 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:52:45,709 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-11T20:52:45,709 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-11T20:52:45,712 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T20:52:45,714 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T20:52:45,714 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:45,715 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:45,715 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T20:52:45,717 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T20:52:45,717 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:45,718 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:45,718 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T20:52:45,720 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T20:52:45,720 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:45,722 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:45,722 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T20:52:45,723 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T20:52:45,724 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:45,725 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T20:52:45,725 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T20:52:45,727 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/hbase/meta/1588230740 2024-11-11T20:52:45,731 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/hbase/meta/1588230740 2024-11-11T20:52:45,734 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T20:52:45,734 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T20:52:45,735 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T20:52:45,738 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T20:52:45,740 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72824143, jitterRate=0.08516429364681244}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T20:52:45,740 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-11T20:52:45,742 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731358365709Writing region info on filesystem at 1731358365709Initializing all the Stores at 1731358365711 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358365711Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358365712 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358365712Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731358365712Cleaning up temporary data from old regions at 1731358365734 (+22 ms)Running coprocessor post-open hooks at 1731358365740 (+6 ms)Region opened successfully at 1731358365742 (+2 ms) 2024-11-11T20:52:45,749 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731358365647 2024-11-11T20:52:45,766 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:45,769 DEBUG [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T20:52:45,770 INFO [RS_OPEN_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-11T20:52:45,771 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 51ca66f7ee3c,44669,1731358362066, state=OPEN 2024-11-11T20:52:45,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44669-0x100308f60b00001, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:52:45,774 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T20:52:45,774 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:52:45,774 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T20:52:45,774 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:45,780 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-11T20:52:45,781 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=51ca66f7ee3c,44669,1731358362066 in 293 msec 2024-11-11T20:52:45,787 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-11T20:52:45,788 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 501 msec 2024-11-11T20:52:45,789 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T20:52:45,790 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-11T20:52:45,811 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:52:45,812 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:52:45,831 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:52:45,834 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51311, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:52:45,856 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1520 sec 2024-11-11T20:52:45,856 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731358365856, completionTime=-1 2024-11-11T20:52:45,860 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-11T20:52:45,860 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T20:52:45,889 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-11T20:52:45,889 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731358425889 2024-11-11T20:52:45,890 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731358485889 2024-11-11T20:52:45,890 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 29 msec 2024-11-11T20:52:45,892 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,38429,1731358361462-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:45,892 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,38429,1731358361462-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:45,892 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,38429,1731358361462-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:45,894 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-51ca66f7ee3c:38429, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:45,894 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:45,895 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:45,900 DEBUG [master/51ca66f7ee3c:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-11T20:52:45,924 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.733sec 2024-11-11T20:52:45,926 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-11T20:52:45,927 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-11T20:52:45,928 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-11T20:52:45,929 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-11T20:52:45,929 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-11T20:52:45,930 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,38429,1731358361462-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T20:52:45,930 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,38429,1731358361462-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-11T20:52:45,939 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-11T20:52:45,940 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-11T20:52:45,940 INFO [master/51ca66f7ee3c:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=51ca66f7ee3c,38429,1731358361462-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T20:52:45,965 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45544f78, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:45,967 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-11T20:52:45,967 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-11T20:52:45,970 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:52:45,972 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:52:45,981 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:52:45,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:52:45,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:52:45,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b270b83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:45,984 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:52:46,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:52:46,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:46,007 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:52:46,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2de67725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:46,011 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:52:46,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:52:46,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:52:46,023 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49266, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:52:46,051 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=51ca66f7ee3c,38429,1731358361462 2024-11-11T20:52:46,064 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=256, ProcessCount=11, AvailableMemoryMB=3315 2024-11-11T20:52:46,074 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T20:52:46,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 51ca66f7ee3c,38429,1731358361462 2024-11-11T20:52:46,082 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@54ffeb4e 2024-11-11T20:52:46,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T20:52:46,088 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T20:52:46,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T20:52:46,095 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T20:52:46,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T20:52:46,106 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T20:52:46,108 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-11T20:52:46,109 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:46,112 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T20:52:46,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T20:52:46,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741835_1011 (size=1061) 2024-11-11T20:52:46,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T20:52:46,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T20:52:46,529 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23 2024-11-11T20:52:46,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741836_1012 (size=53) 2024-11-11T20:52:46,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T20:52:46,942 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:52:46,942 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 09438051ac0c3c1f634095c776d91044, disabling compactions & flushes 2024-11-11T20:52:46,942 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:46,942 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:46,942 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. after waiting 0 ms 2024-11-11T20:52:46,943 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:46,943 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:46,943 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 09438051ac0c3c1f634095c776d91044: Waiting for close lock at 1731358366942Disabling compacts and flushes for region at 1731358366942Disabling writes for close at 1731358366942Writing region close event to WAL at 1731358366943 (+1 ms)Closed at 1731358366943 2024-11-11T20:52:46,946 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T20:52:46,954 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731358366947"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358366947"}]},"ts":"1731358366947"} 2024-11-11T20:52:46,959 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T20:52:46,961 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T20:52:46,964 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358366962"}]},"ts":"1731358366962"} 2024-11-11T20:52:46,968 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T20:52:46,970 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=09438051ac0c3c1f634095c776d91044, ASSIGN}] 2024-11-11T20:52:46,973 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=09438051ac0c3c1f634095c776d91044, ASSIGN 2024-11-11T20:52:46,974 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=09438051ac0c3c1f634095c776d91044, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,44669,1731358362066; forceNewPlan=false, retain=false 2024-11-11T20:52:47,127 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=09438051ac0c3c1f634095c776d91044, regionState=OPENING, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:47,135 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=09438051ac0c3c1f634095c776d91044, ASSIGN because future has completed 2024-11-11T20:52:47,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066}] 2024-11-11T20:52:47,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T20:52:47,299 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:47,300 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:52:47,300 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 09438051ac0c3c1f634095c776d91044 2024-11-11T20:52:47,301 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:52:47,301 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 09438051ac0c3c1f634095c776d91044 2024-11-11T20:52:47,301 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 09438051ac0c3c1f634095c776d91044 2024-11-11T20:52:47,304 INFO [StoreOpener-09438051ac0c3c1f634095c776d91044-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 09438051ac0c3c1f634095c776d91044 2024-11-11T20:52:47,309 INFO [StoreOpener-09438051ac0c3c1f634095c776d91044-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T20:52:47,309 INFO [StoreOpener-09438051ac0c3c1f634095c776d91044-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 09438051ac0c3c1f634095c776d91044 columnFamilyName A 2024-11-11T20:52:47,309 DEBUG [StoreOpener-09438051ac0c3c1f634095c776d91044-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:47,311 INFO [StoreOpener-09438051ac0c3c1f634095c776d91044-1 {}] regionserver.HStore(327): Store=09438051ac0c3c1f634095c776d91044/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:52:47,311 INFO [StoreOpener-09438051ac0c3c1f634095c776d91044-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 09438051ac0c3c1f634095c776d91044 2024-11-11T20:52:47,314 INFO [StoreOpener-09438051ac0c3c1f634095c776d91044-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T20:52:47,314 INFO [StoreOpener-09438051ac0c3c1f634095c776d91044-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 09438051ac0c3c1f634095c776d91044 columnFamilyName B 2024-11-11T20:52:47,315 DEBUG [StoreOpener-09438051ac0c3c1f634095c776d91044-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:47,316 INFO [StoreOpener-09438051ac0c3c1f634095c776d91044-1 {}] regionserver.HStore(327): Store=09438051ac0c3c1f634095c776d91044/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:52:47,316 INFO [StoreOpener-09438051ac0c3c1f634095c776d91044-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 09438051ac0c3c1f634095c776d91044 2024-11-11T20:52:47,318 INFO [StoreOpener-09438051ac0c3c1f634095c776d91044-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T20:52:47,318 INFO [StoreOpener-09438051ac0c3c1f634095c776d91044-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 09438051ac0c3c1f634095c776d91044 columnFamilyName C 2024-11-11T20:52:47,319 DEBUG [StoreOpener-09438051ac0c3c1f634095c776d91044-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:52:47,319 INFO [StoreOpener-09438051ac0c3c1f634095c776d91044-1 {}] regionserver.HStore(327): Store=09438051ac0c3c1f634095c776d91044/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:52:47,320 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:47,320 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 09438051ac0c3c1f634095c776d91044 2024-11-11T20:52:47,322 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044 2024-11-11T20:52:47,322 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044 2024-11-11T20:52:47,325 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 09438051ac0c3c1f634095c776d91044 2024-11-11T20:52:47,325 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 09438051ac0c3c1f634095c776d91044 2024-11-11T20:52:47,326 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T20:52:47,329 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 09438051ac0c3c1f634095c776d91044 2024-11-11T20:52:47,334 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:52:47,335 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 09438051ac0c3c1f634095c776d91044; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70849190, jitterRate=0.05573520064353943}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T20:52:47,335 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 09438051ac0c3c1f634095c776d91044 2024-11-11T20:52:47,336 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 09438051ac0c3c1f634095c776d91044: Running coprocessor pre-open hook at 1731358367301Writing region info on filesystem at 1731358367301Initializing all the Stores at 1731358367304 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358367304Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358367304Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358367304Cleaning up temporary data from old regions at 1731358367325 (+21 ms)Running coprocessor post-open hooks at 1731358367335 (+10 ms)Region opened successfully at 1731358367336 (+1 ms) 2024-11-11T20:52:47,338 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., pid=6, masterSystemTime=1731358367291 2024-11-11T20:52:47,342 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:47,342 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:47,344 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=09438051ac0c3c1f634095c776d91044, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:47,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 because future has completed 2024-11-11T20:52:47,358 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-11T20:52:47,358 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 in 216 msec 2024-11-11T20:52:47,363 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-11T20:52:47,364 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=09438051ac0c3c1f634095c776d91044, ASSIGN in 388 msec 2024-11-11T20:52:47,365 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T20:52:47,365 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358367365"}]},"ts":"1731358367365"} 2024-11-11T20:52:47,369 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T20:52:47,371 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T20:52:47,374 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2730 sec 2024-11-11T20:52:48,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T20:52:48,269 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:52:48,279 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19b2c994, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,279 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:52:48,280 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:52:48,282 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:52:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:52:48,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:52:48,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5583d40c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:52:48,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:52:48,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:48,288 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:52:48,290 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@665358ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,291 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:52:48,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:52:48,295 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:52:48,299 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:52:48,302 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T20:52:48,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 51ca66f7ee3c,38429,1731358361462 2024-11-11T20:52:48,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@32b26fa3 2024-11-11T20:52:48,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T20:52:48,312 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33600, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T20:52:48,318 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a52ffa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,318 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:52:48,318 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:52:48,320 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:52:48,321 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:52:48,321 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:52:48,321 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@149abd79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,321 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:52:48,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:52:48,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:48,324 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33614, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:52:48,325 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a0960a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,329 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51f14cda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,329 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:52:48,329 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:52:48,331 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:52:48,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:52:48,331 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:52:48,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@528027bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:52:48,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:52:48,332 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:48,334 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33622, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:52:48,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@749fb54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,336 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4201ea29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,336 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:52:48,336 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:52:48,337 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:52:48,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:52:48,338 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:52:48,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596d06e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:52:48,339 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:52:48,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:48,343 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33644, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:52:48,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33bdb6a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,344 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45cc41cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,345 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:52:48,345 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:52:48,346 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:52:48,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:52:48,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:52:48,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa64271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,347 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:52:48,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:52:48,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:48,350 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:52:48,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76e341cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,352 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2900b22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,352 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:52:48,352 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:52:48,353 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:52:48,353 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:52:48,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:52:48,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@588e13f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:52:48,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:52:48,354 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:48,355 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33690, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:52:48,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f5127c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,359 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d39686f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,359 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:52:48,359 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:52:48,360 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:52:48,360 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:52:48,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:52:48,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55a9c3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:52:48,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:52:48,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:48,363 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33704, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:52:48,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b096a35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,368 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6963d59b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,368 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:52:48,368 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:52:48,370 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:52:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:52:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:52:48,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ec64ade, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:52:48,371 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:52:48,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:48,373 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33726, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:52:48,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e58d725, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,376 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b9a31d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,376 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:52:48,376 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:52:48,377 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:52:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:52:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:52:48,378 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8c24f6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:52:48,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:52:48,379 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:48,381 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33752, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:52:48,382 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52e2fecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,383 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f0b495b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,383 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:52:48,383 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:52:48,384 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:52:48,385 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:52:48,385 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:52:48,385 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@311f9e8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,385 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:52:48,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:52:48,386 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:52:48,387 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33776, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:52:48,388 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f696c27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:52:48,393 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:52:48,393 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:52:48,395 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:52:48,395 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:52:48,396 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:52:48,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:52:48,397 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:52:48,398 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:52:48,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:52:48,398 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49274, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:52:48,399 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:52:48,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:52:48,400 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:52:48,401 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:52:48,401 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:52:48,402 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:52:48,403 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49294, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:52:48,403 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:52:48,404 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:52:48,406 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:52:48,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:52:48,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:52:48,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:52:48,414 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:52:48,418 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:52:48,418 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:52:48,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:52:48,420 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:52:48,423 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49316, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:52:48,424 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:52:48,424 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:52:48,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:52:48,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:52:48,428 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:52:48,431 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49330, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:52:48,431 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:52:48,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:52:48,434 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:52:48,435 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49338, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:52:48,437 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:52:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:52:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:52:48,439 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:52:48,440 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49350, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:52:48,446 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:52:48,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:52:48,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-11T20:52:48,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T20:52:48,451 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:52:48,453 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:52:48,455 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:52:48,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09438051ac0c3c1f634095c776d91044 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T20:52:48,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 09438051ac0c3c1f634095c776d91044 2024-11-11T20:52:48,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=A 2024-11-11T20:52:48,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:52:48,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=B 2024-11-11T20:52:48,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:52:48,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=C 2024-11-11T20:52:48,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:52:48,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T20:52:48,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T20:52:48,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:48,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:48,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:48,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:48,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:48,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/df5d55d214954f5d94aaa6c231a7c319 is 50, key is test_row_0/A:col10/1731358368482/Put/seqid=0 2024-11-11T20:52:48,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:48,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741837_1013 (size=12001) 2024-11-11T20:52:48,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/df5d55d214954f5d94aaa6c231a7c319 2024-11-11T20:52:48,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358368642, max timestamp=1731358368721Num uniques 859; 2024-11-11T20:52:48,734 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358368642, max timestamp=1731358368721Num uniques 859; 2024-11-11T20:52:48,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:52:48,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:52:48,749 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358368642, max timestamp=1731358368721Num uniques 859; 2024-11-11T20:52:48,750 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:52:48,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T20:52:48,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T20:52:48,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:48,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:48,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358428802, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:48,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:48,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/8fad264debb04c268ba99f6e4f75b0fd is 50, key is test_row_0/B:col10/1731358368482/Put/seqid=0 2024-11-11T20:52:48,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358428808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:48,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358428811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:48,833 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:48,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358428808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:48,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:48,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:48,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:48,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:48,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:48,852 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:48,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:48,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358428826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:48,873 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:48,874 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:48,873 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:48,874 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:48,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:48,875 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:48,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:48,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:48,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:48,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:48,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:48,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:48,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:48,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:48,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:48,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741838_1014 (size=12001) 2024-11-11T20:52:48,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:48,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358428989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:48,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:48,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:48,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:49,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:49,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358428990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:49,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:49,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:49,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358428990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:49,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,004 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:49,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:49,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358428989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:49,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,009 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:49,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T20:52:49,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:49,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358428997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,015 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:49,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:49,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:49,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:49,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:49,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:49,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:49,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T20:52:49,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T20:52:49,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:49,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:49,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:49,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:49,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:49,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:49,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:49,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358429197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:49,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:49,221 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:49,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358429206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:49,229 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,230 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:49,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:49,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358429217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:49,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:49,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358429217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:49,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:49,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:49,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358429227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:49,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,249 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:49,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/8fad264debb04c268ba99f6e4f75b0fd 2024-11-11T20:52:49,338 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T20:52:49,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:49,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:49,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:49,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:49,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:49,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:49,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/6fd5a18b8cba4c25908d8bde486d2f04 is 50, key is test_row_0/C:col10/1731358368482/Put/seqid=0 2024-11-11T20:52:49,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741839_1015 (size=12001) 2024-11-11T20:52:49,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/6fd5a18b8cba4c25908d8bde486d2f04 2024-11-11T20:52:49,430 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/df5d55d214954f5d94aaa6c231a7c319 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/df5d55d214954f5d94aaa6c231a7c319 2024-11-11T20:52:49,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/df5d55d214954f5d94aaa6c231a7c319, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T20:52:49,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/8fad264debb04c268ba99f6e4f75b0fd as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/8fad264debb04c268ba99f6e4f75b0fd 2024-11-11T20:52:49,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/8fad264debb04c268ba99f6e4f75b0fd, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T20:52:49,478 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/6fd5a18b8cba4c25908d8bde486d2f04 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/6fd5a18b8cba4c25908d8bde486d2f04 2024-11-11T20:52:49,491 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/6fd5a18b8cba4c25908d8bde486d2f04, entries=150, sequenceid=13, filesize=11.7 K 2024-11-11T20:52:49,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T20:52:49,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:49,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:49,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 09438051ac0c3c1f634095c776d91044 in 982ms, sequenceid=13, compaction requested=false 2024-11-11T20:52:49,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:49,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:49,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:49,500 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T20:52:49,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:49,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:52:49,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 09438051ac0c3c1f634095c776d91044 2024-11-11T20:52:49,544 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09438051ac0c3c1f634095c776d91044 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-11T20:52:49,545 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=A 2024-11-11T20:52:49,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:52:49,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=B 2024-11-11T20:52:49,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:52:49,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=C 2024-11-11T20:52:49,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:52:49,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/109d35677c5a4aaea40cbefc41087018 is 50, key is test_row_0/A:col10/1731358368820/Put/seqid=0 2024-11-11T20:52:49,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741840_1016 (size=14341) 2024-11-11T20:52:49,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T20:52:49,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358429575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:49,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,609 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:49,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:49,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358429605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:49,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,631 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:49,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:49,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358429606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:49,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,635 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:49,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:49,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358429609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:49,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,636 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:49,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358429614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:49,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:49,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T20:52:49,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:49,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:49,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:49,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:49,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:49,725 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358429716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:49,726 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:49,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:49,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358429737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:49,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,747 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:49,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:49,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358429746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:49,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:49,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:49,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358429747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:49,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:49,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358429756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:49,762 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,763 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:49,817 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T20:52:49,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:49,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:49,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:49,818 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:49,818 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:49,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:49,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:49,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358429937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:49,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,946 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:49,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358429960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:49,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:49,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:49,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358429967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:49,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,972 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:49,972 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:49,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358429967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,973 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:49,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:49,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358429968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:49,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:49,974 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:49,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T20:52:49,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:49,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:49,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:49,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:49,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:49,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:49,985 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/109d35677c5a4aaea40cbefc41087018 2024-11-11T20:52:50,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/6f7ccff10acc4528a8df0aeba4ef881a is 50, key is test_row_0/B:col10/1731358368820/Put/seqid=0 2024-11-11T20:52:50,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741841_1017 (size=12001) 2024-11-11T20:52:50,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T20:52:50,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:50,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:50,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:50,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:50,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:50,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:50,260 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:50,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358430256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:50,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:50,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358430267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:50,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:50,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:50,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:50,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T20:52:50,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:50,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:50,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:50,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:50,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358430286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:50,289 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:50,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:50,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:50,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:50,290 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:50,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:50,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:50,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358430287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:50,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:50,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:50,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:50,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:50,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358430287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:50,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:50,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:50,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:50,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/6f7ccff10acc4528a8df0aeba4ef881a 2024-11-11T20:52:50,444 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T20:52:50,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:50,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:50,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:50,445 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:50,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:50,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:50,470 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/571d458708e746a18ce50136078577b0 is 50, key is test_row_0/C:col10/1731358368820/Put/seqid=0 2024-11-11T20:52:50,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741842_1018 (size=12001) 2024-11-11T20:52:50,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/571d458708e746a18ce50136078577b0 2024-11-11T20:52:50,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/109d35677c5a4aaea40cbefc41087018 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/109d35677c5a4aaea40cbefc41087018 2024-11-11T20:52:50,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/109d35677c5a4aaea40cbefc41087018, entries=200, sequenceid=48, filesize=14.0 K 2024-11-11T20:52:50,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/6f7ccff10acc4528a8df0aeba4ef881a as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/6f7ccff10acc4528a8df0aeba4ef881a 2024-11-11T20:52:50,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/6f7ccff10acc4528a8df0aeba4ef881a, entries=150, sequenceid=48, filesize=11.7 K 2024-11-11T20:52:50,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/571d458708e746a18ce50136078577b0 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/571d458708e746a18ce50136078577b0 2024-11-11T20:52:50,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T20:52:50,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T20:52:50,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:50,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:50,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:50,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:50,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:50,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:50,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/571d458708e746a18ce50136078577b0, entries=150, sequenceid=48, filesize=11.7 K 2024-11-11T20:52:50,604 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=60.38 KB/61830 for 09438051ac0c3c1f634095c776d91044 in 1061ms, sequenceid=48, compaction requested=false 2024-11-11T20:52:50,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:52:50,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T20:52:50,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:50,756 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 09438051ac0c3c1f634095c776d91044 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T20:52:50,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=A 2024-11-11T20:52:50,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:52:50,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=B 2024-11-11T20:52:50,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:52:50,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=C 2024-11-11T20:52:50,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:52:50,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/faea3aa34cec4215b371a4aaf7c30adf is 50, key is test_row_0/A:col10/1731358369606/Put/seqid=0 2024-11-11T20:52:50,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 09438051ac0c3c1f634095c776d91044 2024-11-11T20:52:50,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:50,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741843_1019 (size=12001) 2024-11-11T20:52:50,811 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/faea3aa34cec4215b371a4aaf7c30adf 2024-11-11T20:52:50,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/ab14cf231dc34302bb2409951a840918 is 50, key is test_row_0/B:col10/1731358369606/Put/seqid=0 2024-11-11T20:52:50,852 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14144, totalHeapSize=32376, min timestamp=1731358370812, max timestamp=1731358370837Num uniques 859; 2024-11-11T20:52:50,853 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=14144, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T20:52:50,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358370832, max timestamp=1731358370837Num uniques 859; 2024-11-11T20:52:50,856 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:52:50,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741844_1020 (size=12001) 2024-11-11T20:52:50,862 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/ab14cf231dc34302bb2409951a840918 2024-11-11T20:52:50,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731358370832, max timestamp=1731358370837Num uniques 859; 2024-11-11T20:52:50,876 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-11T20:52:50,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/dcf88198ea23431a81d464a996e12116 is 50, key is test_row_0/C:col10/1731358369606/Put/seqid=0 2024-11-11T20:52:50,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741845_1021 (size=12001) 2024-11-11T20:52:50,893 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/dcf88198ea23431a81d464a996e12116 2024-11-11T20:52:50,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/faea3aa34cec4215b371a4aaf7c30adf as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/faea3aa34cec4215b371a4aaf7c30adf 2024-11-11T20:52:50,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:50,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358430897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:50,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:50,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:50,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:50,916 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358430901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:50,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:50,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:50,918 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:50,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:50,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358430901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:50,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358430901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:50,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:50,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:50,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:50,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:50,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:50,921 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:50,923 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:50,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358430910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:50,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:50,929 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/faea3aa34cec4215b371a4aaf7c30adf, entries=150, sequenceid=60, filesize=11.7 K 2024-11-11T20:52:50,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/ab14cf231dc34302bb2409951a840918 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/ab14cf231dc34302bb2409951a840918 2024-11-11T20:52:50,952 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/ab14cf231dc34302bb2409951a840918, entries=150, sequenceid=60, filesize=11.7 K 2024-11-11T20:52:50,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/dcf88198ea23431a81d464a996e12116 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/dcf88198ea23431a81d464a996e12116 2024-11-11T20:52:50,972 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/dcf88198ea23431a81d464a996e12116, entries=150, sequenceid=60, filesize=11.7 K 2024-11-11T20:52:50,975 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.37 KB/123255 for 09438051ac0c3c1f634095c776d91044 in 219ms, sequenceid=60, compaction requested=true 2024-11-11T20:52:50,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:52:50,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:50,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-11T20:52:50,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-11T20:52:50,983 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-11T20:52:50,983 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5230 sec 2024-11-11T20:52:50,992 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.5440 sec 2024-11-11T20:52:51,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 09438051ac0c3c1f634095c776d91044 2024-11-11T20:52:51,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09438051ac0c3c1f634095c776d91044 3/3 column families, dataSize=133.78 KB heapSize=379.38 KB 2024-11-11T20:52:51,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=A 2024-11-11T20:52:51,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:52:51,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=B 2024-11-11T20:52:51,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:52:51,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=C 2024-11-11T20:52:51,034 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:52:51,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/456ef4e2fcf546a9a3f00274027d7e08 is 50, key is test_row_0/A:col10/1731358371029/Put/seqid=0 2024-11-11T20:52:51,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:51,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358431046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:51,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:51,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:51,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358431048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:51,054 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:51,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358431049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:51,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:51,055 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,056 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:51,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741846_1022 (size=14341) 2024-11-11T20:52:51,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:51,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358431052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,062 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:51,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358431052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,062 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:51,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/456ef4e2fcf546a9a3f00274027d7e08 2024-11-11T20:52:51,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/1f1ffbf7e2474bfa990bb6255f6a4306 is 50, key is test_row_0/B:col10/1731358371029/Put/seqid=0 2024-11-11T20:52:51,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741847_1023 (size=12001) 2024-11-11T20:52:51,141 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/1f1ffbf7e2474bfa990bb6255f6a4306 2024-11-11T20:52:51,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:51,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358431156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:51,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:51,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:51,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358431167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:51,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/4256db5113814bb780dfaf267f56cce2 is 50, key is test_row_0/C:col10/1731358371029/Put/seqid=0 2024-11-11T20:52:51,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,173 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:51,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:51,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358431167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:51,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:51,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:51,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358431169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:51,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:51,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358431169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:51,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:51,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:51,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741848_1024 (size=12001) 2024-11-11T20:52:51,221 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T20:52:51,317 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-11T20:52:51,377 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:51,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358431376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:51,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:51,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:51,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358431377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:51,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,379 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:51,390 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:51,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358431386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:51,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:51,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358431386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:51,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:51,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:51,392 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358431387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:51,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:51,609 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/4256db5113814bb780dfaf267f56cce2 2024-11-11T20:52:51,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/456ef4e2fcf546a9a3f00274027d7e08 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/456ef4e2fcf546a9a3f00274027d7e08 2024-11-11T20:52:51,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/456ef4e2fcf546a9a3f00274027d7e08, entries=200, sequenceid=97, filesize=14.0 K 2024-11-11T20:52:51,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/1f1ffbf7e2474bfa990bb6255f6a4306 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/1f1ffbf7e2474bfa990bb6255f6a4306 2024-11-11T20:52:51,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/1f1ffbf7e2474bfa990bb6255f6a4306, entries=150, sequenceid=97, filesize=11.7 K 2024-11-11T20:52:51,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/4256db5113814bb780dfaf267f56cce2 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/4256db5113814bb780dfaf267f56cce2 2024-11-11T20:52:51,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/4256db5113814bb780dfaf267f56cce2, entries=150, sequenceid=97, filesize=11.7 K 2024-11-11T20:52:51,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.49 KB/143865, heapSize ~399.26 KB/408840, currentSize=40.25 KB/41220 for 09438051ac0c3c1f634095c776d91044 in 651ms, sequenceid=97, compaction requested=true 2024-11-11T20:52:51,683 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:52:51,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09438051ac0c3c1f634095c776d91044:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:52:51,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:52:51,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09438051ac0c3c1f634095c776d91044:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:52:51,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:52:51,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09438051ac0c3c1f634095c776d91044:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:52:51,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:52:51,688 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:52:51,689 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:52:51,694 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:52:51,696 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 09438051ac0c3c1f634095c776d91044/B is initiating minor compaction (all files) 2024-11-11T20:52:51,697 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09438051ac0c3c1f634095c776d91044/B in TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:51,699 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/8fad264debb04c268ba99f6e4f75b0fd, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/6f7ccff10acc4528a8df0aeba4ef881a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/ab14cf231dc34302bb2409951a840918, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/1f1ffbf7e2474bfa990bb6255f6a4306] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp, totalSize=46.9 K 2024-11-11T20:52:51,700 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:52:51,701 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 09438051ac0c3c1f634095c776d91044/A is initiating minor compaction (all files) 2024-11-11T20:52:51,701 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09438051ac0c3c1f634095c776d91044/A in TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:51,701 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/df5d55d214954f5d94aaa6c231a7c319, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/109d35677c5a4aaea40cbefc41087018, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/faea3aa34cec4215b371a4aaf7c30adf, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/456ef4e2fcf546a9a3f00274027d7e08] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp, totalSize=51.4 K 2024-11-11T20:52:51,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 09438051ac0c3c1f634095c776d91044 2024-11-11T20:52:51,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09438051ac0c3c1f634095c776d91044 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T20:52:51,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=A 2024-11-11T20:52:51,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:52:51,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=B 2024-11-11T20:52:51,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:52:51,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=C 2024-11-11T20:52:51,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:52:51,703 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 8fad264debb04c268ba99f6e4f75b0fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731358368482 2024-11-11T20:52:51,704 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 6f7ccff10acc4528a8df0aeba4ef881a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731358368763 2024-11-11T20:52:51,707 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting df5d55d214954f5d94aaa6c231a7c319, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731358368482 2024-11-11T20:52:51,708 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting ab14cf231dc34302bb2409951a840918, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731358369548 2024-11-11T20:52:51,711 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f1ffbf7e2474bfa990bb6255f6a4306, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731358371027 2024-11-11T20:52:51,712 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 109d35677c5a4aaea40cbefc41087018, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731358368763 2024-11-11T20:52:51,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/b70bcce1ca714428bddbec7be843b08a is 50, key is test_row_0/A:col10/1731358371042/Put/seqid=0 2024-11-11T20:52:51,718 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting faea3aa34cec4215b371a4aaf7c30adf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731358369548 2024-11-11T20:52:51,727 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 456ef4e2fcf546a9a3f00274027d7e08, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731358370894 2024-11-11T20:52:51,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741849_1025 (size=16681) 2024-11-11T20:52:51,759 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/b70bcce1ca714428bddbec7be843b08a 2024-11-11T20:52:51,787 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09438051ac0c3c1f634095c776d91044#A#compaction#13 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:52:51,790 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/022cc05cae03401282f86f4a44ffdb51 is 50, key is test_row_0/A:col10/1731358371029/Put/seqid=0 2024-11-11T20:52:51,800 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T20:52:51,800 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-11T20:52:51,802 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09438051ac0c3c1f634095c776d91044#B#compaction#14 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:52:51,803 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T20:52:51,803 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-11T20:52:51,803 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/948b8b95adb54efe937a2e9cd2021c61 is 50, key is test_row_0/B:col10/1731358371029/Put/seqid=0 2024-11-11T20:52:51,803 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-11T20:52:51,803 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-11T20:52:51,804 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T20:52:51,804 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-11T20:52:51,810 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/8f1ff165af6848e5af216fb1254d4cec is 50, key is test_row_0/B:col10/1731358371042/Put/seqid=0 2024-11-11T20:52:51,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358371754, max timestamp=1731358371805Num uniques 859; 2024-11-11T20:52:51,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:52:51,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358371754, max timestamp=1731358371806Num uniques 859; 2024-11-11T20:52:51,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:52:51,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741850_1026 (size=12139) 2024-11-11T20:52:51,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358371754, max timestamp=1731358371805Num uniques 859; 2024-11-11T20:52:51,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:52:51,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741851_1027 (size=12139) 2024-11-11T20:52:51,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741852_1028 (size=12001) 2024-11-11T20:52:51,873 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/948b8b95adb54efe937a2e9cd2021c61 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/948b8b95adb54efe937a2e9cd2021c61 2024-11-11T20:52:51,923 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 09438051ac0c3c1f634095c776d91044/B of 09438051ac0c3c1f634095c776d91044 into 948b8b95adb54efe937a2e9cd2021c61(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:52:51,923 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:52:51,926 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., storeName=09438051ac0c3c1f634095c776d91044/B, priority=12, startTime=1731358371687; duration=0sec 2024-11-11T20:52:51,927 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:52:51,927 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09438051ac0c3c1f634095c776d91044:B 2024-11-11T20:52:51,928 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:52:51,931 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:52:51,931 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 09438051ac0c3c1f634095c776d91044/C is initiating minor compaction (all files) 2024-11-11T20:52:51,931 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09438051ac0c3c1f634095c776d91044/C in TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:51,931 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/6fd5a18b8cba4c25908d8bde486d2f04, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/571d458708e746a18ce50136078577b0, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/dcf88198ea23431a81d464a996e12116, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/4256db5113814bb780dfaf267f56cce2] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp, totalSize=46.9 K 2024-11-11T20:52:51,932 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fd5a18b8cba4c25908d8bde486d2f04, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731358368482 2024-11-11T20:52:51,933 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 571d458708e746a18ce50136078577b0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731358368763 2024-11-11T20:52:51,933 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting dcf88198ea23431a81d464a996e12116, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731358369548 2024-11-11T20:52:51,934 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 4256db5113814bb780dfaf267f56cce2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731358371027 2024-11-11T20:52:51,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:51,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358431950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:51,961 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09438051ac0c3c1f634095c776d91044#C#compaction#16 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:52:51,962 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/c05b4314b8e14591b36d31a703402991 is 50, key is test_row_0/C:col10/1731358371029/Put/seqid=0 2024-11-11T20:52:51,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358431954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,964 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,965 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:51,968 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:51,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358431956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:51,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:51,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:51,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358431956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:51,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741853_1029 (size=12139) 2024-11-11T20:52:51,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:51,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358431967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:51,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:51,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:51,996 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/c05b4314b8e14591b36d31a703402991 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/c05b4314b8e14591b36d31a703402991 2024-11-11T20:52:52,010 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 09438051ac0c3c1f634095c776d91044/C of 09438051ac0c3c1f634095c776d91044 into c05b4314b8e14591b36d31a703402991(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:52:52,010 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:52:52,010 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., storeName=09438051ac0c3c1f634095c776d91044/C, priority=12, startTime=1731358371688; duration=0sec 2024-11-11T20:52:52,011 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:52:52,011 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09438051ac0c3c1f634095c776d91044:C 2024-11-11T20:52:52,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:52,085 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:52,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358432077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:52,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358432077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:52,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:52,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358432077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:52,086 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:52,087 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:52,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358432077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:52,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,088 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:52,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:52,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358432087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:52,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:52,258 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/022cc05cae03401282f86f4a44ffdb51 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/022cc05cae03401282f86f4a44ffdb51 2024-11-11T20:52:52,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/8f1ff165af6848e5af216fb1254d4cec 2024-11-11T20:52:52,274 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 09438051ac0c3c1f634095c776d91044/A of 09438051ac0c3c1f634095c776d91044 into 022cc05cae03401282f86f4a44ffdb51(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:52:52,274 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:52:52,274 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., storeName=09438051ac0c3c1f634095c776d91044/A, priority=12, startTime=1731358371684; duration=0sec 2024-11-11T20:52:52,274 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:52:52,275 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09438051ac0c3c1f634095c776d91044:A 2024-11-11T20:52:52,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/86b3501685f741e0929419b509a50024 is 50, key is test_row_0/C:col10/1731358371042/Put/seqid=0 2024-11-11T20:52:52,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358432296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:52,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741854_1030 (size=12001) 2024-11-11T20:52:52,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358432296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:52,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:52,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:52,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358432296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:52,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/86b3501685f741e0929419b509a50024 2024-11-11T20:52:52,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:52,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358432296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:52,304 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:52,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358432296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:52,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,305 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:52,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/b70bcce1ca714428bddbec7be843b08a as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/b70bcce1ca714428bddbec7be843b08a 2024-11-11T20:52:52,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/b70bcce1ca714428bddbec7be843b08a, entries=250, sequenceid=108, filesize=16.3 K 2024-11-11T20:52:52,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/8f1ff165af6848e5af216fb1254d4cec as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/8f1ff165af6848e5af216fb1254d4cec 2024-11-11T20:52:52,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/8f1ff165af6848e5af216fb1254d4cec, entries=150, sequenceid=108, filesize=11.7 K 2024-11-11T20:52:52,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/86b3501685f741e0929419b509a50024 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/86b3501685f741e0929419b509a50024 2024-11-11T20:52:52,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/86b3501685f741e0929419b509a50024, entries=150, sequenceid=108, filesize=11.7 K 2024-11-11T20:52:52,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 09438051ac0c3c1f634095c776d91044 in 661ms, sequenceid=108, compaction requested=false 2024-11-11T20:52:52,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:52:52,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T20:52:52,607 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:52:52,607 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:52:52,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-11T20:52:52,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731358371920, max timestamp=1731358372608Num uniques 1018; 2024-11-11T20:52:52,614 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-11T20:52:52,616 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731358371920, max timestamp=1731358372611Num uniques 1018; 2024-11-11T20:52:52,617 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-11T20:52:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 09438051ac0c3c1f634095c776d91044 2024-11-11T20:52:52,617 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09438051ac0c3c1f634095c776d91044 3/3 column families, dataSize=76.04 KB heapSize=208.88 KB 2024-11-11T20:52:52,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731358371920, max timestamp=1731358372611Num uniques 1018; 2024-11-11T20:52:52,618 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-11T20:52:52,618 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:52:52,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T20:52:52,620 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:52:52,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=A 2024-11-11T20:52:52,620 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:52:52,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:52:52,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=B 2024-11-11T20:52:52,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:52:52,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=C 2024-11-11T20:52:52,621 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:52:52,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/dac4e0169f8e40ffa09d5d3f6d345242 is 50, key is test_row_0/A:col10/1731358371953/Put/seqid=0 2024-11-11T20:52:52,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741855_1031 (size=12151) 2024-11-11T20:52:52,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/dac4e0169f8e40ffa09d5d3f6d345242 2024-11-11T20:52:52,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358372648, max timestamp=1731358372668Num uniques 859; 2024-11-11T20:52:52,673 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358372648, max timestamp=1731358372668Num uniques 859; 2024-11-11T20:52:52,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:52:52,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:52:52,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358372648, max timestamp=1731358372668Num uniques 859; 2024-11-11T20:52:52,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:52:52,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/983c4eea0c92447a9ea37b34a1823314 is 50, key is test_row_0/B:col10/1731358371953/Put/seqid=0 2024-11-11T20:52:52,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741856_1032 (size=12151) 2024-11-11T20:52:52,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/983c4eea0c92447a9ea37b34a1823314 2024-11-11T20:52:52,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T20:52:52,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:52,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358432720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:52,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:52,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:52,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358432723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:52,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:52,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358432723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:52,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731358372716, max timestamp=1731358372722Num uniques 1068; 2024-11-11T20:52:52,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,729 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,730 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:52,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-11T20:52:52,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731358372716, max timestamp=1731358372722Num uniques 1068; 2024-11-11T20:52:52,730 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-11T20:52:52,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,734 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:52,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731358372716, max timestamp=1731358372722Num uniques 1068; 2024-11-11T20:52:52,735 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-11T20:52:52,736 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/a5bbfc8e2e114a00a6a31bd98d0678db is 50, key is test_row_0/C:col10/1731358371953/Put/seqid=0 2024-11-11T20:52:52,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741857_1033 (size=12151) 2024-11-11T20:52:52,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=150 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/a5bbfc8e2e114a00a6a31bd98d0678db 2024-11-11T20:52:52,775 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T20:52:52,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:52,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:52,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:52,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:52,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:52,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/dac4e0169f8e40ffa09d5d3f6d345242 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/dac4e0169f8e40ffa09d5d3f6d345242 2024-11-11T20:52:52,792 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/dac4e0169f8e40ffa09d5d3f6d345242, entries=150, sequenceid=150, filesize=11.9 K 2024-11-11T20:52:52,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/983c4eea0c92447a9ea37b34a1823314 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/983c4eea0c92447a9ea37b34a1823314 2024-11-11T20:52:52,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/983c4eea0c92447a9ea37b34a1823314, entries=150, sequenceid=150, filesize=11.9 K 2024-11-11T20:52:52,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/a5bbfc8e2e114a00a6a31bd98d0678db as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/a5bbfc8e2e114a00a6a31bd98d0678db 2024-11-11T20:52:52,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731358372766, max timestamp=1731358372809Num uniques 1027; 2024-11-11T20:52:52,814 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731358372766, max timestamp=1731358372809Num uniques 1027; 2024-11-11T20:52:52,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731358372766, max timestamp=1731358372809Num uniques 1027; 2024-11-11T20:52:52,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-11T20:52:52,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-11T20:52:52,815 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-11T20:52:52,824 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/a5bbfc8e2e114a00a6a31bd98d0678db, entries=150, sequenceid=150, filesize=11.9 K 2024-11-11T20:52:52,826 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~100.50 KB/102912, currentSize=40.25 KB/41220 for 09438051ac0c3c1f634095c776d91044 in 209ms, sequenceid=150, compaction requested=true 2024-11-11T20:52:52,826 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:52:52,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09438051ac0c3c1f634095c776d91044:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:52:52,827 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:52:52,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:52:52,827 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:52:52,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09438051ac0c3c1f634095c776d91044:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:52:52,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:52:52,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09438051ac0c3c1f634095c776d91044:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:52:52,827 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:52:52,830 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:52:52,830 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 09438051ac0c3c1f634095c776d91044/B is initiating minor compaction (all files) 2024-11-11T20:52:52,830 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09438051ac0c3c1f634095c776d91044/B in TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:52,830 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/948b8b95adb54efe937a2e9cd2021c61, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/8f1ff165af6848e5af216fb1254d4cec, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/983c4eea0c92447a9ea37b34a1823314] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp, totalSize=35.4 K 2024-11-11T20:52:52,831 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40971 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:52:52,831 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 09438051ac0c3c1f634095c776d91044/A is initiating minor compaction (all files) 2024-11-11T20:52:52,831 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09438051ac0c3c1f634095c776d91044/A in TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:52,831 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/022cc05cae03401282f86f4a44ffdb51, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/b70bcce1ca714428bddbec7be843b08a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/dac4e0169f8e40ffa09d5d3f6d345242] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp, totalSize=40.0 K 2024-11-11T20:52:52,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 09438051ac0c3c1f634095c776d91044 2024-11-11T20:52:52,833 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 948b8b95adb54efe937a2e9cd2021c61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731358371027 2024-11-11T20:52:52,833 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 022cc05cae03401282f86f4a44ffdb51, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731358371027 2024-11-11T20:52:52,834 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting b70bcce1ca714428bddbec7be843b08a, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731358371042 2024-11-11T20:52:52,835 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f1ff165af6848e5af216fb1254d4cec, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731358371042 2024-11-11T20:52:52,835 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 983c4eea0c92447a9ea37b34a1823314, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731358371953 2024-11-11T20:52:52,836 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting dac4e0169f8e40ffa09d5d3f6d345242, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731358371953 2024-11-11T20:52:52,837 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09438051ac0c3c1f634095c776d91044 3/3 column families, dataSize=53.67 KB heapSize=139.55 KB 2024-11-11T20:52:52,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=A 2024-11-11T20:52:52,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:52:52,845 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=B 2024-11-11T20:52:52,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:52:52,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=C 2024-11-11T20:52:52,846 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:52:52,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/dde2029dbb55466e90d9071fcfa9c66a is 50, key is test_row_0/A:col10/1731358372835/Put/seqid=0 2024-11-11T20:52:52,864 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09438051ac0c3c1f634095c776d91044#B#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:52:52,864 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09438051ac0c3c1f634095c776d91044#A#compaction#23 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:52:52,865 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/d0df97357d614170a5acc4444fd919e6 is 50, key is test_row_0/A:col10/1731358371953/Put/seqid=0 2024-11-11T20:52:52,869 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/2ef10c144cfb4363841404cc71645f6a is 50, key is test_row_0/B:col10/1731358371953/Put/seqid=0 2024-11-11T20:52:52,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741858_1034 (size=12151) 2024-11-11T20:52:52,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/dde2029dbb55466e90d9071fcfa9c66a 2024-11-11T20:52:52,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741860_1036 (size=12391) 2024-11-11T20:52:52,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731358372879, max timestamp=1731358372897Num uniques 859; 2024-11-11T20:52:52,902 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T20:52:52,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731358372879, max timestamp=1731358372897Num uniques 859; 2024-11-11T20:52:52,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=309, cellSize=14145, totalHeapSize=32376, min timestamp=1731358372879, max timestamp=1731358372897Num uniques 859; 2024-11-11T20:52:52,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T20:52:52,903 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=14145, suffix heap size=117064,new segment heap size=32376 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=309 2024-11-11T20:52:52,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741859_1035 (size=12391) 2024-11-11T20:52:52,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/b11c1cf0e79446d79936022edd3920ec is 50, key is test_row_0/B:col10/1731358372835/Put/seqid=0 2024-11-11T20:52:52,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T20:52:52,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T20:52:52,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:52,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:52,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:52,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:52,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:52,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:52,951 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/d0df97357d614170a5acc4444fd919e6 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/d0df97357d614170a5acc4444fd919e6 2024-11-11T20:52:52,953 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/2ef10c144cfb4363841404cc71645f6a as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/2ef10c144cfb4363841404cc71645f6a 2024-11-11T20:52:52,968 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09438051ac0c3c1f634095c776d91044/A of 09438051ac0c3c1f634095c776d91044 into d0df97357d614170a5acc4444fd919e6(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:52:52,968 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:52:52,968 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., storeName=09438051ac0c3c1f634095c776d91044/A, priority=13, startTime=1731358372827; duration=0sec 2024-11-11T20:52:52,968 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:52:52,968 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09438051ac0c3c1f634095c776d91044:A 2024-11-11T20:52:52,969 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:52:52,971 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09438051ac0c3c1f634095c776d91044/B of 09438051ac0c3c1f634095c776d91044 into 2ef10c144cfb4363841404cc71645f6a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:52:52,971 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:52:52,972 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., storeName=09438051ac0c3c1f634095c776d91044/B, priority=13, startTime=1731358372827; duration=0sec 2024-11-11T20:52:52,972 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:52:52,972 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09438051ac0c3c1f634095c776d91044:B 2024-11-11T20:52:52,972 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:52:52,972 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 09438051ac0c3c1f634095c776d91044/C is initiating minor compaction (all files) 2024-11-11T20:52:52,972 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09438051ac0c3c1f634095c776d91044/C in TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:52,973 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/c05b4314b8e14591b36d31a703402991, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/86b3501685f741e0929419b509a50024, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/a5bbfc8e2e114a00a6a31bd98d0678db] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp, totalSize=35.4 K 2024-11-11T20:52:52,974 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting c05b4314b8e14591b36d31a703402991, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1731358371027 2024-11-11T20:52:52,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:52,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358432961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:52,976 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,977 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:52,978 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86b3501685f741e0929419b509a50024, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731358371042 2024-11-11T20:52:52,979 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5bbfc8e2e114a00a6a31bd98d0678db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731358371953 2024-11-11T20:52:52,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741861_1037 (size=12151) 2024-11-11T20:52:52,983 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/b11c1cf0e79446d79936022edd3920ec 2024-11-11T20:52:52,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358432974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:52,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:52,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,995 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:52,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358432975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:52,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:52,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:52,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358432979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:52,996 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:52,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358432975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:52,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,996 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,996 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:52,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:52,997 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:53,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/5463bfcd5f8b44d3bb259a878c57a163 is 50, key is test_row_0/C:col10/1731358372835/Put/seqid=0 2024-11-11T20:52:53,012 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09438051ac0c3c1f634095c776d91044#C#compaction#26 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:52:53,013 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/3b0d9d08303f459e974f97357ddc2fec is 50, key is test_row_0/C:col10/1731358371953/Put/seqid=0 2024-11-11T20:52:53,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741862_1038 (size=12151) 2024-11-11T20:52:53,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741863_1039 (size=12391) 2024-11-11T20:52:53,052 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/3b0d9d08303f459e974f97357ddc2fec as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/3b0d9d08303f459e974f97357ddc2fec 2024-11-11T20:52:53,068 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09438051ac0c3c1f634095c776d91044/C of 09438051ac0c3c1f634095c776d91044 into 3b0d9d08303f459e974f97357ddc2fec(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:52:53,068 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:52:53,068 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., storeName=09438051ac0c3c1f634095c776d91044/C, priority=13, startTime=1731358372827; duration=0sec 2024-11-11T20:52:53,068 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:52:53,068 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09438051ac0c3c1f634095c776d91044:C 2024-11-11T20:52:53,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:53,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358433086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:53,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,094 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:53,106 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T20:52:53,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:53,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:53,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:53,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:53,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:53,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:53,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:53,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358433106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:53,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:53,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:53,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358433106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:53,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:53,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:53,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358433106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:53,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:53,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358433106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:53,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:53,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:53,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T20:52:53,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T20:52:53,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:53,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:53,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:53,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:53,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:53,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:53,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:53,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358433306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:53,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,309 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:53,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:53,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358433316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:53,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:53,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358433316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:53,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:53,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358433316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:53,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358433317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,325 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:53,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T20:52:53,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:53,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:53,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:53,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:53,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:53,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:53,427 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/5463bfcd5f8b44d3bb259a878c57a163 2024-11-11T20:52:53,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/dde2029dbb55466e90d9071fcfa9c66a as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/dde2029dbb55466e90d9071fcfa9c66a 2024-11-11T20:52:53,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/dde2029dbb55466e90d9071fcfa9c66a, entries=150, sequenceid=209, filesize=11.9 K 2024-11-11T20:52:53,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/b11c1cf0e79446d79936022edd3920ec as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/b11c1cf0e79446d79936022edd3920ec 2024-11-11T20:52:53,478 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/b11c1cf0e79446d79936022edd3920ec, entries=150, sequenceid=209, filesize=11.9 K 2024-11-11T20:52:53,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/5463bfcd5f8b44d3bb259a878c57a163 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/5463bfcd5f8b44d3bb259a878c57a163 2024-11-11T20:52:53,495 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/5463bfcd5f8b44d3bb259a878c57a163, entries=150, sequenceid=209, filesize=11.9 K 2024-11-11T20:52:53,497 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~159.42 KB/163248, currentSize=140.89 KB/144270 for 09438051ac0c3c1f634095c776d91044 in 661ms, sequenceid=209, compaction requested=false 2024-11-11T20:52:53,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:52:53,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T20:52:53,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:53,569 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 09438051ac0c3c1f634095c776d91044 3/3 column families, dataSize=140.89 KB heapSize=390.84 KB 2024-11-11T20:52:53,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=A 2024-11-11T20:52:53,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:52:53,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=B 2024-11-11T20:52:53,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:52:53,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=C 2024-11-11T20:52:53,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:52:53,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/d5e983c9f6c9473cab1245449beb0103 is 50, key is test_row_0/A:col10/1731358372956/Put/seqid=0 2024-11-11T20:52:53,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741864_1040 (size=12151) 2024-11-11T20:52:53,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:53,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 09438051ac0c3c1f634095c776d91044 2024-11-11T20:52:53,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:53,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358433637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:53,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,645 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,646 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:53,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:53,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358433639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:53,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:53,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:53,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358433645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:53,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:53,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358433650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:53,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:53,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358433658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:53,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,665 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:53,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,666 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:53,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T20:52:53,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:53,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358433757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:53,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:53,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,762 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:53,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358433757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:53,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:53,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:53,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358433766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:53,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,769 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:53,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:53,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358433777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:53,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:53,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,783 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:53,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358433777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:53,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,783 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:53,967 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:53,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358433966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:53,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:53,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358433966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:53,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,969 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:53,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358433976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:53,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:53,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358433986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:53,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:53,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358433986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:53,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:53,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:53,993 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/d5e983c9f6c9473cab1245449beb0103 2024-11-11T20:52:54,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/3fbb6265cacb469386225212a4ffa8e8 is 50, key is test_row_0/B:col10/1731358372956/Put/seqid=0 2024-11-11T20:52:54,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741865_1041 (size=12151) 2024-11-11T20:52:54,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358434276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:54,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:54,279 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:54,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:54,280 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358434276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:54,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:54,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:54,280 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:54,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:54,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358434286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:54,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:54,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:54,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:54,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358434295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:54,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:54,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:54,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:54,299 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358434295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:54,299 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:54,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:54,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:54,434 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/3fbb6265cacb469386225212a4ffa8e8 2024-11-11T20:52:54,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/e0fa4a2a8849471888178cd41252c9f8 is 50, key is test_row_0/C:col10/1731358372956/Put/seqid=0 2024-11-11T20:52:54,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741866_1042 (size=12151) 2024-11-11T20:52:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T20:52:54,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358434786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:54,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:54,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:54,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:54,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358434786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:54,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:54,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:54,788 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:54,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358434796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:54,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:54,802 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:54,803 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:54,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358434805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:54,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:54,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:54,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:54,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358434805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:54,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:54,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:54,811 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:54,860 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/e0fa4a2a8849471888178cd41252c9f8 2024-11-11T20:52:54,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/d5e983c9f6c9473cab1245449beb0103 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/d5e983c9f6c9473cab1245449beb0103 2024-11-11T20:52:54,875 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/d5e983c9f6c9473cab1245449beb0103, entries=150, sequenceid=247, filesize=11.9 K 2024-11-11T20:52:54,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/3fbb6265cacb469386225212a4ffa8e8 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/3fbb6265cacb469386225212a4ffa8e8 2024-11-11T20:52:54,887 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/3fbb6265cacb469386225212a4ffa8e8, entries=150, sequenceid=247, filesize=11.9 K 2024-11-11T20:52:54,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/e0fa4a2a8849471888178cd41252c9f8 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/e0fa4a2a8849471888178cd41252c9f8 2024-11-11T20:52:54,900 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/e0fa4a2a8849471888178cd41252c9f8, entries=150, sequenceid=247, filesize=11.9 K 2024-11-11T20:52:54,902 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~390.80 KB/400176, currentSize=60.38 KB/61830 for 09438051ac0c3c1f634095c776d91044 in 1333ms, sequenceid=247, compaction requested=true 2024-11-11T20:52:54,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:52:54,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:54,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-11T20:52:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-11T20:52:54,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-11T20:52:54,910 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2870 sec 2024-11-11T20:52:54,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 2.3050 sec 2024-11-11T20:52:55,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 09438051ac0c3c1f634095c776d91044 2024-11-11T20:52:55,802 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09438051ac0c3c1f634095c776d91044 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T20:52:55,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=A 2024-11-11T20:52:55,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:52:55,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=B 2024-11-11T20:52:55,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:52:55,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=C 2024-11-11T20:52:55,810 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:52:55,831 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/631cc022a7824fd7b8ae36f54dd91f09 is 50, key is test_row_0/A:col10/1731358373642/Put/seqid=0 2024-11-11T20:52:55,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741867_1043 (size=14641) 2024-11-11T20:52:55,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358435859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:55,887 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:55,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:55,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:55,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358435862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:55,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:55,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:55,895 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:55,904 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:55,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358435887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:55,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:55,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:55,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358435887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:55,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:55,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:55,907 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:55,911 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:55,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358435898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:55,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:55,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:55,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:56,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:56,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358435996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:56,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:56,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:56,006 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:56,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:56,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358436006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:56,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:56,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:56,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:56,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:56,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358436016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:56,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:56,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358436016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:56,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:56,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:56,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:56,051 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358436047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:56,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:56,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:56,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:56,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:56,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358436216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358436217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:56,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:56,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:56,219 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:56,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:56,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:56,220 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:56,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:56,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358436236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:56,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:56,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:56,243 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:56,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358436237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:56,248 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:56,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358436256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:56,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:56,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:56,262 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:56,286 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/631cc022a7824fd7b8ae36f54dd91f09 2024-11-11T20:52:56,313 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/f7bf92c447f2439cbdfe4f17aac33a36 is 50, key is test_row_0/B:col10/1731358373642/Put/seqid=0 2024-11-11T20:52:56,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741868_1044 (size=12201) 2024-11-11T20:52:56,355 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/f7bf92c447f2439cbdfe4f17aac33a36 2024-11-11T20:52:56,391 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/fccef71ef0b14266af28347d55a76180 is 50, key is test_row_0/C:col10/1731358373642/Put/seqid=0 2024-11-11T20:52:56,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741869_1045 (size=12201) 2024-11-11T20:52:56,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=260 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/fccef71ef0b14266af28347d55a76180 2024-11-11T20:52:56,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/631cc022a7824fd7b8ae36f54dd91f09 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/631cc022a7824fd7b8ae36f54dd91f09 2024-11-11T20:52:56,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/631cc022a7824fd7b8ae36f54dd91f09, entries=200, sequenceid=260, filesize=14.3 K 2024-11-11T20:52:56,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/f7bf92c447f2439cbdfe4f17aac33a36 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/f7bf92c447f2439cbdfe4f17aac33a36 2024-11-11T20:52:56,496 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/f7bf92c447f2439cbdfe4f17aac33a36, entries=150, sequenceid=260, filesize=11.9 K 2024-11-11T20:52:56,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/fccef71ef0b14266af28347d55a76180 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/fccef71ef0b14266af28347d55a76180 2024-11-11T20:52:56,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/fccef71ef0b14266af28347d55a76180, entries=150, sequenceid=260, filesize=11.9 K 2024-11-11T20:52:56,509 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 09438051ac0c3c1f634095c776d91044 in 707ms, sequenceid=260, compaction requested=true 2024-11-11T20:52:56,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:52:56,510 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:52:56,513 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51334 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:52:56,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09438051ac0c3c1f634095c776d91044:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:52:56,513 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:52:56,513 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 09438051ac0c3c1f634095c776d91044/A is initiating minor compaction (all files) 2024-11-11T20:52:56,513 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:52:56,513 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09438051ac0c3c1f634095c776d91044/A in TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:56,514 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/d0df97357d614170a5acc4444fd919e6, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/dde2029dbb55466e90d9071fcfa9c66a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/d5e983c9f6c9473cab1245449beb0103, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/631cc022a7824fd7b8ae36f54dd91f09] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp, totalSize=50.1 K 2024-11-11T20:52:56,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09438051ac0c3c1f634095c776d91044:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:52:56,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:52:56,514 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0df97357d614170a5acc4444fd919e6, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731358371953 2024-11-11T20:52:56,514 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09438051ac0c3c1f634095c776d91044:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:52:56,515 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:52:56,515 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting dde2029dbb55466e90d9071fcfa9c66a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731358372766 2024-11-11T20:52:56,516 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5e983c9f6c9473cab1245449beb0103, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731358372956 2024-11-11T20:52:56,516 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48894 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:52:56,516 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 09438051ac0c3c1f634095c776d91044/B is initiating minor compaction (all files) 2024-11-11T20:52:56,516 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09438051ac0c3c1f634095c776d91044/B in TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:56,516 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/2ef10c144cfb4363841404cc71645f6a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/b11c1cf0e79446d79936022edd3920ec, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/3fbb6265cacb469386225212a4ffa8e8, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/f7bf92c447f2439cbdfe4f17aac33a36] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp, totalSize=47.7 K 2024-11-11T20:52:56,517 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 631cc022a7824fd7b8ae36f54dd91f09, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731358373629 2024-11-11T20:52:56,518 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ef10c144cfb4363841404cc71645f6a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731358371953 2024-11-11T20:52:56,519 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting b11c1cf0e79446d79936022edd3920ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731358372766 2024-11-11T20:52:56,521 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fbb6265cacb469386225212a4ffa8e8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731358372956 2024-11-11T20:52:56,522 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting f7bf92c447f2439cbdfe4f17aac33a36, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731358373629 2024-11-11T20:52:56,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 09438051ac0c3c1f634095c776d91044 2024-11-11T20:52:56,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358375846, max timestamp=1731358376536Num uniques 859; 2024-11-11T20:52:56,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09438051ac0c3c1f634095c776d91044 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-11T20:52:56,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:52:56,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358375846, max timestamp=1731358376536Num uniques 859; 2024-11-11T20:52:56,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:52:56,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358375846, max timestamp=1731358376536Num uniques 859; 2024-11-11T20:52:56,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:52:56,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=A 2024-11-11T20:52:56,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:52:56,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=B 2024-11-11T20:52:56,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:52:56,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=C 2024-11-11T20:52:56,547 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:52:56,554 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09438051ac0c3c1f634095c776d91044#A#compaction#33 average throughput is 0.66 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:52:56,555 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/f5323228b7794195b90fd74f735ab3a8 is 50, key is test_row_0/A:col10/1731358373642/Put/seqid=0 2024-11-11T20:52:56,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/569cbba743b5463d82abf94da4d06d69 is 50, key is test_row_0/A:col10/1731358375873/Put/seqid=0 2024-11-11T20:52:56,566 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09438051ac0c3c1f634095c776d91044#B#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:52:56,567 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/290f72a9031b488c84438e0e197470a2 is 50, key is test_row_0/B:col10/1731358373642/Put/seqid=0 2024-11-11T20:52:56,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741870_1046 (size=12577) 2024-11-11T20:52:56,601 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/f5323228b7794195b90fd74f735ab3a8 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/f5323228b7794195b90fd74f735ab3a8 2024-11-11T20:52:56,617 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 09438051ac0c3c1f634095c776d91044/A of 09438051ac0c3c1f634095c776d91044 into f5323228b7794195b90fd74f735ab3a8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:52:56,617 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:52:56,617 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., storeName=09438051ac0c3c1f634095c776d91044/A, priority=12, startTime=1731358376510; duration=0sec 2024-11-11T20:52:56,617 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:52:56,617 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09438051ac0c3c1f634095c776d91044:A 2024-11-11T20:52:56,617 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:52:56,620 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48894 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:52:56,620 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 09438051ac0c3c1f634095c776d91044/C is initiating minor compaction (all files) 2024-11-11T20:52:56,620 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09438051ac0c3c1f634095c776d91044/C in TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:56,620 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/3b0d9d08303f459e974f97357ddc2fec, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/5463bfcd5f8b44d3bb259a878c57a163, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/e0fa4a2a8849471888178cd41252c9f8, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/fccef71ef0b14266af28347d55a76180] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp, totalSize=47.7 K 2024-11-11T20:52:56,621 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b0d9d08303f459e974f97357ddc2fec, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=150, earliestPutTs=1731358371953 2024-11-11T20:52:56,623 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5463bfcd5f8b44d3bb259a878c57a163, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1731358372766 2024-11-11T20:52:56,624 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0fa4a2a8849471888178cd41252c9f8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731358372956 2024-11-11T20:52:56,625 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting fccef71ef0b14266af28347d55a76180, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731358373629 2024-11-11T20:52:56,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9564, totalHeapSize=21976, min timestamp=1731358376612, max timestamp=1731358376627Num uniques 859; 2024-11-11T20:52:56,630 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=9564, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:52:56,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358376612, max timestamp=1731358376627Num uniques 859; 2024-11-11T20:52:56,634 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:52:56,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358376612, max timestamp=1731358376627Num uniques 859; 2024-11-11T20:52:56,639 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:52:56,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741872_1048 (size=17181) 2024-11-11T20:52:56,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741871_1047 (size=12577) 2024-11-11T20:52:56,655 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09438051ac0c3c1f634095c776d91044#C#compaction#36 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:52:56,656 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/14e67390edcb427a98343dd80f3b6c30 is 50, key is test_row_0/C:col10/1731358373642/Put/seqid=0 2024-11-11T20:52:56,666 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/290f72a9031b488c84438e0e197470a2 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/290f72a9031b488c84438e0e197470a2 2024-11-11T20:52:56,679 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 09438051ac0c3c1f634095c776d91044/B of 09438051ac0c3c1f634095c776d91044 into 290f72a9031b488c84438e0e197470a2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:52:56,679 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:52:56,680 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., storeName=09438051ac0c3c1f634095c776d91044/B, priority=12, startTime=1731358376513; duration=0sec 2024-11-11T20:52:56,680 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:52:56,681 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09438051ac0c3c1f634095c776d91044:B 2024-11-11T20:52:56,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731358376667, max timestamp=1731358376716Num uniques 1068; 2024-11-11T20:52:56,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731358376667, max timestamp=1731358376716Num uniques 1068; 2024-11-11T20:52:56,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-11T20:52:56,721 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-11T20:52:56,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741873_1049 (size=12577) 2024-11-11T20:52:56,724 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731358376667, max timestamp=1731358376716Num uniques 1068; 2024-11-11T20:52:56,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-11T20:52:56,734 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/14e67390edcb427a98343dd80f3b6c30 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/14e67390edcb427a98343dd80f3b6c30 2024-11-11T20:52:56,752 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 09438051ac0c3c1f634095c776d91044/C of 09438051ac0c3c1f634095c776d91044 into 14e67390edcb427a98343dd80f3b6c30(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:52:56,752 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:52:56,753 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., storeName=09438051ac0c3c1f634095c776d91044/C, priority=12, startTime=1731358376514; duration=0sec 2024-11-11T20:52:56,753 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:52:56,753 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09438051ac0c3c1f634095c776d91044:C 2024-11-11T20:52:56,792 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1731358376737, max timestamp=1731358376788Num uniques 1077; 2024-11-11T20:52:56,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1731358376737, max timestamp=1731358376788Num uniques 1077; 2024-11-11T20:52:56,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-11-11T20:52:56,793 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=14972, suffix heap size=139976,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=327 2024-11-11T20:52:56,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731358376737, max timestamp=1731358376788Num uniques 1077; 2024-11-11T20:52:56,796 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=8102, suffix heap size=139976,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=177 2024-11-11T20:52:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T20:52:56,799 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:52:56,799 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:52:56,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-11T20:52:56,803 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:52:56,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T20:52:56,804 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:52:56,804 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:52:56,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731358376812, max timestamp=1731358376864Num uniques 1186; 2024-11-11T20:52:56,871 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=15386, suffix heap size=151312,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=336 2024-11-11T20:52:56,873 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731358376812, max timestamp=1731358376864Num uniques 1186; 2024-11-11T20:52:56,874 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=15386, suffix heap size=151312,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=336 2024-11-11T20:52:56,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=336, cellSize=15386, totalHeapSize=35184, min timestamp=1731358376812, max timestamp=1731358376865Num uniques 1036; 2024-11-11T20:52:56,875 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=15386, suffix heap size=135712,new segment heap size=35184 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=336 2024-11-11T20:52:56,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T20:52:56,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T20:52:56,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:56,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:56,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:56,960 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:56,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:56,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:56,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358436938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:56,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:56,977 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731358376928, max timestamp=1731358376961Num uniques 1195; 2024-11-11T20:52:56,977 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731358376928, max timestamp=1731358376964Num uniques 1195; 2024-11-11T20:52:56,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731358376928, max timestamp=1731358376964Num uniques 1195; 2024-11-11T20:52:56,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=8930, suffix heap size=152248,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=195 2024-11-11T20:52:56,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=8930, suffix heap size=152248,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=195 2024-11-11T20:52:56,978 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=54730, new segment data size=8930, suffix heap size=152248,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1195, new segment cells count=195 2024-11-11T20:52:56,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358436951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:56,979 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:57,039 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/569cbba743b5463d82abf94da4d06d69 2024-11-11T20:52:57,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/2ec52e5dd11d43e9950a7e7406e5eb97 is 50, key is test_row_0/B:col10/1731358375873/Put/seqid=0 2024-11-11T20:52:57,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T20:52:57,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741874_1050 (size=12301) 2024-11-11T20:52:57,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T20:52:57,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:57,143 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/2ec52e5dd11d43e9950a7e7406e5eb97 2024-11-11T20:52:57,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:57,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:57,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:57,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:57,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:57,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731358377073, max timestamp=1731358377138Num uniques 1054; 2024-11-11T20:52:57,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731358377073, max timestamp=1731358377138Num uniques 1054; 2024-11-11T20:52:57,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-11T20:52:57,148 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-11T20:52:57,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731358377073, max timestamp=1731358377138Num uniques 1054; 2024-11-11T20:52:57,152 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-11T20:52:57,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/37e60aa8fa734564a19a0834d3c0f71f is 50, key is test_row_0/C:col10/1731358375873/Put/seqid=0 2024-11-11T20:52:57,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731358377185, max timestamp=1731358377214Num uniques 1013; 2024-11-11T20:52:57,218 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731358377185, max timestamp=1731358377214Num uniques 1013; 2024-11-11T20:52:57,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-11T20:52:57,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-11T20:52:57,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=163, cellSize=7458, totalHeapSize=17192, min timestamp=1731358377185, max timestamp=1731358377214Num uniques 1013; 2024-11-11T20:52:57,219 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=7458, suffix heap size=133320,new segment heap size=17192 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=163 2024-11-11T20:52:57,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741875_1051 (size=12301) 2024-11-11T20:52:57,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=282 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/37e60aa8fa734564a19a0834d3c0f71f 2024-11-11T20:52:57,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/569cbba743b5463d82abf94da4d06d69 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/569cbba743b5463d82abf94da4d06d69 2024-11-11T20:52:57,252 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/569cbba743b5463d82abf94da4d06d69, entries=250, sequenceid=282, filesize=16.8 K 2024-11-11T20:52:57,255 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/2ec52e5dd11d43e9950a7e7406e5eb97 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/2ec52e5dd11d43e9950a7e7406e5eb97 2024-11-11T20:52:57,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/2ec52e5dd11d43e9950a7e7406e5eb97, entries=150, sequenceid=282, filesize=12.0 K 2024-11-11T20:52:57,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/37e60aa8fa734564a19a0834d3c0f71f as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/37e60aa8fa734564a19a0834d3c0f71f 2024-11-11T20:52:57,283 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/37e60aa8fa734564a19a0834d3c0f71f, entries=150, sequenceid=282, filesize=12.0 K 2024-11-11T20:52:57,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=127.47 KB/130530 for 09438051ac0c3c1f634095c776d91044 in 748ms, sequenceid=282, compaction requested=false 2024-11-11T20:52:57,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:52:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 09438051ac0c3c1f634095c776d91044 2024-11-11T20:52:57,291 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09438051ac0c3c1f634095c776d91044 3/3 column families, dataSize=134.18 KB heapSize=384.61 KB 2024-11-11T20:52:57,293 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731358377251, max timestamp=1731358377289Num uniques 1022; 2024-11-11T20:52:57,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=12452, suffix heap size=134256,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=272 2024-11-11T20:52:57,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731358377251, max timestamp=1731358377289Num uniques 1022; 2024-11-11T20:52:57,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=272, cellSize=12452, totalHeapSize=28528, min timestamp=1731358377251, max timestamp=1731358377289Num uniques 1022; 2024-11-11T20:52:57,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=12452, suffix heap size=134256,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=272 2024-11-11T20:52:57,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46802, new segment data size=12452, suffix heap size=134256,new segment heap size=28528 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1022, new segment cells count=272 2024-11-11T20:52:57,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=A 2024-11-11T20:52:57,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:52:57,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=B 2024-11-11T20:52:57,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:52:57,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=C 2024-11-11T20:52:57,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:52:57,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T20:52:57,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:57,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:57,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:57,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:57,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:57,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/8a0719cf2ad74deebfbf0acbd9036af0 is 50, key is test_row_0/A:col10/1731358377279/Put/seqid=0 2024-11-11T20:52:57,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741876_1052 (size=19621) 2024-11-11T20:52:57,361 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/8a0719cf2ad74deebfbf0acbd9036af0 2024-11-11T20:52:57,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/c16104b7816548f1a2681fa2a7b3d9e5 is 50, key is test_row_0/B:col10/1731358377279/Put/seqid=0 2024-11-11T20:52:57,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358377378, max timestamp=1731358377411Num uniques 859; 2024-11-11T20:52:57,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358377378, max timestamp=1731358377411Num uniques 859; 2024-11-11T20:52:57,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:52:57,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:52:57,415 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358377378, max timestamp=1731358377411Num uniques 859; 2024-11-11T20:52:57,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:52:57,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T20:52:57,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741877_1053 (size=12301) 2024-11-11T20:52:57,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T20:52:57,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:57,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:57,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:57,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:57,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:57,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:57,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731358377475, max timestamp=1731358377505Num uniques 1068; 2024-11-11T20:52:57,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731358377475, max timestamp=1731358377505Num uniques 1068; 2024-11-11T20:52:57,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-11T20:52:57,510 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-11T20:52:57,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731358377475, max timestamp=1731358377505Num uniques 1068; 2024-11-11T20:52:57,512 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-11T20:52:57,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1731358377531, max timestamp=1731358377571Num uniques 1077; 2024-11-11T20:52:57,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1731358377531, max timestamp=1731358377571Num uniques 1077; 2024-11-11T20:52:57,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=17262, suffix heap size=139976,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=377 2024-11-11T20:52:57,577 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=17262, suffix heap size=139976,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=377 2024-11-11T20:52:57,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1731358377531, max timestamp=1731358377571Num uniques 1077; 2024-11-11T20:52:57,581 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=17262, suffix heap size=139976,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=377 2024-11-11T20:52:57,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T20:52:57,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:57,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:57,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:57,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:57,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:57,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:57,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:57,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358437626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:57,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,639 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:57,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358437631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:57,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:57,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358437631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:57,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358437633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:57,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:57,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358437639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:57,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,651 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:57,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,651 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,652 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:57,652 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:57,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:57,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358437746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:57,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,749 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:57,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:57,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:57,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358437757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:57,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358437757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:57,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:57,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:57,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:57,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358437757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:57,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:57,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358437757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:57,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:57,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T20:52:57,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:57,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:57,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:57,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:57,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:57,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:57,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:57,849 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/c16104b7816548f1a2681fa2a7b3d9e5 2024-11-11T20:52:57,879 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/816fd990fc2b4172be55844eb517fe94 is 50, key is test_row_0/C:col10/1731358377279/Put/seqid=0 2024-11-11T20:52:57,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741878_1054 (size=12301) 2024-11-11T20:52:57,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T20:52:57,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:57,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:57,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:57,924 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:57,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:57,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:57,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T20:52:57,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:57,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358437956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:57,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,965 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:57,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:57,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358437977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:57,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,979 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:57,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:57,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358437977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:57,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358437977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:57,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:57,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:57,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358437977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:57,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:57,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:58,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T20:52:58,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:58,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:58,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:58,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:58,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:58,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:58,233 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T20:52:58,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:58,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:58,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:58,235 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:58,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:58,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:58,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:58,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358438276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:58,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:58,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:58,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:58,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:58,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358438286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:58,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:58,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358438286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:58,294 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:58,294 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:58,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358438290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:58,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:58,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358438287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:58,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:58,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/816fd990fc2b4172be55844eb517fe94 2024-11-11T20:52:58,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/8a0719cf2ad74deebfbf0acbd9036af0 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/8a0719cf2ad74deebfbf0acbd9036af0 2024-11-11T20:52:58,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/8a0719cf2ad74deebfbf0acbd9036af0, entries=300, sequenceid=426, filesize=19.2 K 2024-11-11T20:52:58,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/c16104b7816548f1a2681fa2a7b3d9e5 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/c16104b7816548f1a2681fa2a7b3d9e5 2024-11-11T20:52:58,376 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/c16104b7816548f1a2681fa2a7b3d9e5, entries=150, sequenceid=426, filesize=12.0 K 2024-11-11T20:52:58,377 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/816fd990fc2b4172be55844eb517fe94 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/816fd990fc2b4172be55844eb517fe94 2024-11-11T20:52:58,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T20:52:58,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/816fd990fc2b4172be55844eb517fe94, entries=150, sequenceid=426, filesize=12.0 K 2024-11-11T20:52:58,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:58,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:58,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:58,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:58,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:58,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:52:58,393 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~95.44 KB/97728, currentSize=161.02 KB/164880 for 09438051ac0c3c1f634095c776d91044 in 1103ms, sequenceid=426, compaction requested=true 2024-11-11T20:52:58,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:52:58,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09438051ac0c3c1f634095c776d91044:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:52:58,394 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:52:58,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:52:58,395 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:52:58,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09438051ac0c3c1f634095c776d91044:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:52:58,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:52:58,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09438051ac0c3c1f634095c776d91044:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:52:58,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:52:58,397 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 49379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:52:58,397 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 09438051ac0c3c1f634095c776d91044/A is initiating minor compaction (all files) 2024-11-11T20:52:58,397 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09438051ac0c3c1f634095c776d91044/A in TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:58,397 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/f5323228b7794195b90fd74f735ab3a8, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/569cbba743b5463d82abf94da4d06d69, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/8a0719cf2ad74deebfbf0acbd9036af0] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp, totalSize=48.2 K 2024-11-11T20:52:58,398 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37179 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:52:58,398 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 09438051ac0c3c1f634095c776d91044/B is initiating minor compaction (all files) 2024-11-11T20:52:58,398 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09438051ac0c3c1f634095c776d91044/B in TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:58,398 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/290f72a9031b488c84438e0e197470a2, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/2ec52e5dd11d43e9950a7e7406e5eb97, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/c16104b7816548f1a2681fa2a7b3d9e5] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp, totalSize=36.3 K 2024-11-11T20:52:58,398 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5323228b7794195b90fd74f735ab3a8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731358373629 2024-11-11T20:52:58,399 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 290f72a9031b488c84438e0e197470a2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731358373629 2024-11-11T20:52:58,399 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 569cbba743b5463d82abf94da4d06d69, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731358375846 2024-11-11T20:52:58,399 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ec52e5dd11d43e9950a7e7406e5eb97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731358375873 2024-11-11T20:52:58,400 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a0719cf2ad74deebfbf0acbd9036af0, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731358377251 2024-11-11T20:52:58,400 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting c16104b7816548f1a2681fa2a7b3d9e5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731358377279 2024-11-11T20:52:58,428 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09438051ac0c3c1f634095c776d91044#B#compaction#42 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:52:58,429 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/b66b08156c494898b6c9521d568ab4f0 is 50, key is test_row_0/B:col10/1731358377279/Put/seqid=0 2024-11-11T20:52:58,445 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09438051ac0c3c1f634095c776d91044#A#compaction#43 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:52:58,446 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/2782b7764f4c453888dacdc60c3564cb is 50, key is test_row_0/A:col10/1731358377279/Put/seqid=0 2024-11-11T20:52:58,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741879_1055 (size=12779) 2024-11-11T20:52:58,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741880_1056 (size=12779) 2024-11-11T20:52:58,512 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/2782b7764f4c453888dacdc60c3564cb as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/2782b7764f4c453888dacdc60c3564cb 2024-11-11T20:52:58,524 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09438051ac0c3c1f634095c776d91044/A of 09438051ac0c3c1f634095c776d91044 into 2782b7764f4c453888dacdc60c3564cb(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:52:58,524 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:52:58,524 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., storeName=09438051ac0c3c1f634095c776d91044/A, priority=13, startTime=1731358378394; duration=0sec 2024-11-11T20:52:58,525 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:52:58,525 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09438051ac0c3c1f634095c776d91044:A 2024-11-11T20:52:58,525 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:52:58,528 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37179 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:52:58,528 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 09438051ac0c3c1f634095c776d91044/C is initiating minor compaction (all files) 2024-11-11T20:52:58,528 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09438051ac0c3c1f634095c776d91044/C in TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:58,529 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/14e67390edcb427a98343dd80f3b6c30, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/37e60aa8fa734564a19a0834d3c0f71f, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/816fd990fc2b4172be55844eb517fe94] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp, totalSize=36.3 K 2024-11-11T20:52:58,529 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14e67390edcb427a98343dd80f3b6c30, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=260, earliestPutTs=1731358373629 2024-11-11T20:52:58,531 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37e60aa8fa734564a19a0834d3c0f71f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=282, earliestPutTs=1731358375873 2024-11-11T20:52:58,538 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 816fd990fc2b4172be55844eb517fe94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731358377279 2024-11-11T20:52:58,545 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T20:52:58,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:58,545 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 09438051ac0c3c1f634095c776d91044 3/3 column families, dataSize=161.02 KB heapSize=444.23 KB 2024-11-11T20:52:58,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=A 2024-11-11T20:52:58,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:52:58,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=B 2024-11-11T20:52:58,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:52:58,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=C 2024-11-11T20:52:58,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:52:58,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/9098f4bb663e4b2385dba3146d2c99f2 is 50, key is test_row_0/A:col10/1731358377630/Put/seqid=0 2024-11-11T20:52:58,572 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09438051ac0c3c1f634095c776d91044#C#compaction#45 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:52:58,573 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/5cb7ecf97ca34f68ae3c97f83d678821 is 50, key is test_row_0/C:col10/1731358377279/Put/seqid=0 2024-11-11T20:52:58,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741881_1057 (size=12301) 2024-11-11T20:52:58,587 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/9098f4bb663e4b2385dba3146d2c99f2 2024-11-11T20:52:58,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741882_1058 (size=12779) 2024-11-11T20:52:58,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/b2fa476601fb492297a69f69d46b9147 is 50, key is test_row_0/B:col10/1731358377630/Put/seqid=0 2024-11-11T20:52:58,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741883_1059 (size=12301) 2024-11-11T20:52:58,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 09438051ac0c3c1f634095c776d91044 2024-11-11T20:52:58,788 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:52:58,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:58,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:58,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358438805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:58,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358438806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:58,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:58,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:58,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:58,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358438812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:58,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:58,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:58,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:58,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:58,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:58,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:58,819 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:58,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:58,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358438820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:58,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:58,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:58,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:58,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:58,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358438823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:58,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:58,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:58,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:58,897 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/b66b08156c494898b6c9521d568ab4f0 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/b66b08156c494898b6c9521d568ab4f0 2024-11-11T20:52:58,909 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09438051ac0c3c1f634095c776d91044/B of 09438051ac0c3c1f634095c776d91044 into b66b08156c494898b6c9521d568ab4f0(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:52:58,910 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:52:58,910 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., storeName=09438051ac0c3c1f634095c776d91044/B, priority=13, startTime=1731358378394; duration=0sec 2024-11-11T20:52:58,910 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:52:58,910 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09438051ac0c3c1f634095c776d91044:B 2024-11-11T20:52:58,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:58,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358438925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:58,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:58,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:58,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:58,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:58,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358438936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:58,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:58,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:58,940 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:58,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T20:52:59,006 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/5cb7ecf97ca34f68ae3c97f83d678821 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/5cb7ecf97ca34f68ae3c97f83d678821 2024-11-11T20:52:59,019 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09438051ac0c3c1f634095c776d91044/C of 09438051ac0c3c1f634095c776d91044 into 5cb7ecf97ca34f68ae3c97f83d678821(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:52:59,019 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:52:59,019 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., storeName=09438051ac0c3c1f634095c776d91044/C, priority=13, startTime=1731358378396; duration=0sec 2024-11-11T20:52:59,020 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:52:59,020 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09438051ac0c3c1f634095c776d91044:C 2024-11-11T20:52:59,072 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/b2fa476601fb492297a69f69d46b9147 2024-11-11T20:52:59,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/666d248b632d46d7ab511f601cd14a83 is 50, key is test_row_0/C:col10/1731358377630/Put/seqid=0 2024-11-11T20:52:59,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741884_1060 (size=12301) 2024-11-11T20:52:59,145 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=498 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/666d248b632d46d7ab511f601cd14a83 2024-11-11T20:52:59,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:59,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358439146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:59,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:59,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:59,155 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:59,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:59,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358439146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:59,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:59,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:59,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/9098f4bb663e4b2385dba3146d2c99f2 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/9098f4bb663e4b2385dba3146d2c99f2 2024-11-11T20:52:59,169 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/9098f4bb663e4b2385dba3146d2c99f2, entries=150, sequenceid=498, filesize=12.0 K 2024-11-11T20:52:59,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/b2fa476601fb492297a69f69d46b9147 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/b2fa476601fb492297a69f69d46b9147 2024-11-11T20:52:59,189 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/b2fa476601fb492297a69f69d46b9147, entries=150, sequenceid=498, filesize=12.0 K 2024-11-11T20:52:59,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/666d248b632d46d7ab511f601cd14a83 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/666d248b632d46d7ab511f601cd14a83 2024-11-11T20:52:59,201 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/666d248b632d46d7ab511f601cd14a83, entries=150, sequenceid=498, filesize=12.0 K 2024-11-11T20:52:59,203 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~444.19 KB/454848, currentSize=33.54 KB/34350 for 09438051ac0c3c1f634095c776d91044 in 658ms, sequenceid=498, compaction requested=false 2024-11-11T20:52:59,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:52:59,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:52:59,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-11T20:52:59,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-11T20:52:59,208 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-11T20:52:59,209 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4010 sec 2024-11-11T20:52:59,212 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 2.4110 sec 2024-11-11T20:52:59,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09438051ac0c3c1f634095c776d91044 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T20:52:59,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 09438051ac0c3c1f634095c776d91044 2024-11-11T20:52:59,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=A 2024-11-11T20:52:59,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:52:59,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=B 2024-11-11T20:52:59,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:52:59,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=C 2024-11-11T20:52:59,480 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:52:59,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/fe9d9bfeca6f4313aa405f95adb38103 is 50, key is test_row_0/A:col10/1731358378815/Put/seqid=0 2024-11-11T20:52:59,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741885_1061 (size=12301) 2024-11-11T20:52:59,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358379481, max timestamp=1731358379587Num uniques 859; 2024-11-11T20:52:59,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:52:59,605 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358379481, max timestamp=1731358379587Num uniques 859; 2024-11-11T20:52:59,610 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:52:59,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358379481, max timestamp=1731358379587Num uniques 859; 2024-11-11T20:52:59,619 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:52:59,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:59,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358439747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:59,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:59,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:59,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:59,754 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:59,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358439752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:59,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:59,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:59,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:59,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:59,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358439826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:59,834 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_0' in 09438051ac0c3c1f634095c776d91044 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:52:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:59,836 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:59,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:59,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358439837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:59,839 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in 09438051ac0c3c1f634095c776d91044 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:52:59,840 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:59,841 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:59,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:59,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358439838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:59,842 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_1' in 09438051ac0c3c1f634095c776d91044 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:52:59,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:59,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:59,846 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:59,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:59,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358439856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:59,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:59,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:59,862 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:59,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:52:59,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358439865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:52:59,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:59,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:52:59,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:52:59,951 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/fe9d9bfeca6f4313aa405f95adb38103 2024-11-11T20:52:59,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/366ef1618e764b67b8ea876944cc8c33 is 50, key is test_row_0/B:col10/1731358378815/Put/seqid=0 2024-11-11T20:52:59,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741886_1062 (size=12301) 2024-11-11T20:53:00,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:00,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358440066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:00,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:00,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:00,067 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:00,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:00,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358440075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:00,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:00,078 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:00,079 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:00,377 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/366ef1618e764b67b8ea876944cc8c33 2024-11-11T20:53:00,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:00,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358440376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:00,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:00,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:00,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:00,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/43f25c4495b8422d82529ee1379d690b is 50, key is test_row_0/C:col10/1731358378815/Put/seqid=0 2024-11-11T20:53:00,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358440387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:00,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:00,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741887_1063 (size=12301) 2024-11-11T20:53:00,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=511 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/43f25c4495b8422d82529ee1379d690b 2024-11-11T20:53:00,420 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/fe9d9bfeca6f4313aa405f95adb38103 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/fe9d9bfeca6f4313aa405f95adb38103 2024-11-11T20:53:00,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/fe9d9bfeca6f4313aa405f95adb38103, entries=150, sequenceid=511, filesize=12.0 K 2024-11-11T20:53:00,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/366ef1618e764b67b8ea876944cc8c33 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/366ef1618e764b67b8ea876944cc8c33 2024-11-11T20:53:00,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/366ef1618e764b67b8ea876944cc8c33, entries=150, sequenceid=511, filesize=12.0 K 2024-11-11T20:53:00,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/43f25c4495b8422d82529ee1379d690b as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/43f25c4495b8422d82529ee1379d690b 2024-11-11T20:53:00,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/43f25c4495b8422d82529ee1379d690b, entries=150, sequenceid=511, filesize=12.0 K 2024-11-11T20:53:00,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 09438051ac0c3c1f634095c776d91044 in 985ms, sequenceid=511, compaction requested=true 2024-11-11T20:53:00,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:53:00,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09438051ac0c3c1f634095c776d91044:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:53:00,459 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:00,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:00,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09438051ac0c3c1f634095c776d91044:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:53:00,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:00,459 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:00,459 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09438051ac0c3c1f634095c776d91044:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:53:00,460 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:00,461 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:00,461 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:00,462 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 09438051ac0c3c1f634095c776d91044/A is initiating minor compaction (all files) 2024-11-11T20:53:00,462 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 09438051ac0c3c1f634095c776d91044/B is initiating minor compaction (all files) 2024-11-11T20:53:00,462 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09438051ac0c3c1f634095c776d91044/B in TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:00,462 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09438051ac0c3c1f634095c776d91044/A in TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:00,462 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/b66b08156c494898b6c9521d568ab4f0, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/b2fa476601fb492297a69f69d46b9147, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/366ef1618e764b67b8ea876944cc8c33] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp, totalSize=36.5 K 2024-11-11T20:53:00,462 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/2782b7764f4c453888dacdc60c3564cb, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/9098f4bb663e4b2385dba3146d2c99f2, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/fe9d9bfeca6f4313aa405f95adb38103] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp, totalSize=36.5 K 2024-11-11T20:53:00,463 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting b66b08156c494898b6c9521d568ab4f0, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731358377279 2024-11-11T20:53:00,463 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2782b7764f4c453888dacdc60c3564cb, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731358377279 2024-11-11T20:53:00,464 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting b2fa476601fb492297a69f69d46b9147, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731358377626 2024-11-11T20:53:00,464 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9098f4bb663e4b2385dba3146d2c99f2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731358377626 2024-11-11T20:53:00,464 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 366ef1618e764b67b8ea876944cc8c33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1731358378815 2024-11-11T20:53:00,465 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe9d9bfeca6f4313aa405f95adb38103, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1731358378815 2024-11-11T20:53:00,480 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09438051ac0c3c1f634095c776d91044#B#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:00,481 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/a5000c108f1449cc883ff3fdc0604a21 is 50, key is test_row_0/B:col10/1731358378815/Put/seqid=0 2024-11-11T20:53:00,491 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09438051ac0c3c1f634095c776d91044#A#compaction#52 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:00,492 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/09f55155819f42389b23feb2350f39b9 is 50, key is test_row_0/A:col10/1731358378815/Put/seqid=0 2024-11-11T20:53:00,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741889_1065 (size=12881) 2024-11-11T20:53:00,529 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/09f55155819f42389b23feb2350f39b9 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/09f55155819f42389b23feb2350f39b9 2024-11-11T20:53:00,538 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09438051ac0c3c1f634095c776d91044/A of 09438051ac0c3c1f634095c776d91044 into 09f55155819f42389b23feb2350f39b9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:00,538 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:53:00,539 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., storeName=09438051ac0c3c1f634095c776d91044/A, priority=13, startTime=1731358380459; duration=0sec 2024-11-11T20:53:00,539 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:00,539 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09438051ac0c3c1f634095c776d91044:A 2024-11-11T20:53:00,539 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:00,540 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37381 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:00,541 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 09438051ac0c3c1f634095c776d91044/C is initiating minor compaction (all files) 2024-11-11T20:53:00,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741888_1064 (size=12881) 2024-11-11T20:53:00,541 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09438051ac0c3c1f634095c776d91044/C in TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:00,541 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/5cb7ecf97ca34f68ae3c97f83d678821, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/666d248b632d46d7ab511f601cd14a83, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/43f25c4495b8422d82529ee1379d690b] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp, totalSize=36.5 K 2024-11-11T20:53:00,542 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cb7ecf97ca34f68ae3c97f83d678821, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1731358377279 2024-11-11T20:53:00,544 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 666d248b632d46d7ab511f601cd14a83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=498, earliestPutTs=1731358377626 2024-11-11T20:53:00,545 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43f25c4495b8422d82529ee1379d690b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1731358378815 2024-11-11T20:53:00,549 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/a5000c108f1449cc883ff3fdc0604a21 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/a5000c108f1449cc883ff3fdc0604a21 2024-11-11T20:53:00,558 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09438051ac0c3c1f634095c776d91044/B of 09438051ac0c3c1f634095c776d91044 into a5000c108f1449cc883ff3fdc0604a21(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:00,558 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:53:00,558 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., storeName=09438051ac0c3c1f634095c776d91044/B, priority=13, startTime=1731358380459; duration=0sec 2024-11-11T20:53:00,558 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:00,558 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09438051ac0c3c1f634095c776d91044:B 2024-11-11T20:53:00,580 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09438051ac0c3c1f634095c776d91044#C#compaction#53 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:00,582 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/46d5c775659a46d0bd9ae1d3cd2e6072 is 50, key is test_row_0/C:col10/1731358378815/Put/seqid=0 2024-11-11T20:53:00,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741890_1066 (size=12881) 2024-11-11T20:53:00,614 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/46d5c775659a46d0bd9ae1d3cd2e6072 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/46d5c775659a46d0bd9ae1d3cd2e6072 2024-11-11T20:53:00,626 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09438051ac0c3c1f634095c776d91044/C of 09438051ac0c3c1f634095c776d91044 into 46d5c775659a46d0bd9ae1d3cd2e6072(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:00,626 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:53:00,626 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., storeName=09438051ac0c3c1f634095c776d91044/C, priority=13, startTime=1731358380459; duration=0sec 2024-11-11T20:53:00,626 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:00,626 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09438051ac0c3c1f634095c776d91044:C 2024-11-11T20:53:00,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 09438051ac0c3c1f634095c776d91044 2024-11-11T20:53:00,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09438051ac0c3c1f634095c776d91044 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-11T20:53:00,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=A 2024-11-11T20:53:00,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:00,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=B 2024-11-11T20:53:00,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:00,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=C 2024-11-11T20:53:00,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:00,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/4538847c1c4a4a5e96725747ca79cd8b is 50, key is test_row_0/A:col10/1731358379735/Put/seqid=0 2024-11-11T20:53:00,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741891_1067 (size=14741) 2024-11-11T20:53:00,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=551 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/4538847c1c4a4a5e96725747ca79cd8b 2024-11-11T20:53:00,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/176c60c329514f86b84ae5094e96cb12 is 50, key is test_row_0/B:col10/1731358379735/Put/seqid=0 2024-11-11T20:53:00,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741892_1068 (size=12301) 2024-11-11T20:53:00,923 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=551 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/176c60c329514f86b84ae5094e96cb12 2024-11-11T20:53:00,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:00,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358440932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:00,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:00,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:00,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:00,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358440933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:00,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:00,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:00,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:00,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:00,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/d5d81f4a269b4ad480df2d031f7d73f6 is 50, key is test_row_0/C:col10/1731358379735/Put/seqid=0 2024-11-11T20:53:00,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741893_1069 (size=12301) 2024-11-11T20:53:00,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=551 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/d5d81f4a269b4ad480df2d031f7d73f6 2024-11-11T20:53:00,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/4538847c1c4a4a5e96725747ca79cd8b as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/4538847c1c4a4a5e96725747ca79cd8b 2024-11-11T20:53:00,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T20:53:00,996 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:53:00,997 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:53:00,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-11T20:53:00,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T20:53:01,000 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:53:01,001 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:53:01,001 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:53:01,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/4538847c1c4a4a5e96725747ca79cd8b, entries=200, sequenceid=551, filesize=14.4 K 2024-11-11T20:53:01,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/176c60c329514f86b84ae5094e96cb12 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/176c60c329514f86b84ae5094e96cb12 2024-11-11T20:53:01,021 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/176c60c329514f86b84ae5094e96cb12, entries=150, sequenceid=551, filesize=12.0 K 2024-11-11T20:53:01,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/d5d81f4a269b4ad480df2d031f7d73f6 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/d5d81f4a269b4ad480df2d031f7d73f6 2024-11-11T20:53:01,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/d5d81f4a269b4ad480df2d031f7d73f6, entries=150, sequenceid=551, filesize=12.0 K 2024-11-11T20:53:01,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=53.67 KB/54960 for 09438051ac0c3c1f634095c776d91044 in 151ms, sequenceid=551, compaction requested=false 2024-11-11T20:53:01,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:53:01,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 09438051ac0c3c1f634095c776d91044 2024-11-11T20:53:01,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09438051ac0c3c1f634095c776d91044 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T20:53:01,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=A 2024-11-11T20:53:01,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:01,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=B 2024-11-11T20:53:01,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:01,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=C 2024-11-11T20:53:01,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:01,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/f2c88d22774f434f9712e0e25573ce83 is 50, key is test_row_0/A:col10/1731358381048/Put/seqid=0 2024-11-11T20:53:01,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741894_1070 (size=14741) 2024-11-11T20:53:01,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T20:53:01,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:01,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358441145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:01,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:01,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358441147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:01,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:01,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:01,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:01,151 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:01,151 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:01,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:01,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T20:53:01,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:01,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:53:01,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:01,157 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:01,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:01,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:01,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358441256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:01,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:01,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:01,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:01,263 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:01,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358441256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:01,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:01,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:01,264 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:01,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T20:53:01,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:01,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:53:01,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:01,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:01,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:01,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:01,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T20:53:01,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T20:53:01,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:01,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:53:01,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:01,467 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:01,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:01,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358441466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:01,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:01,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:01,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:01,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358441466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:01,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:01,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:01,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:01,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:01,518 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/f2c88d22774f434f9712e0e25573ce83 2024-11-11T20:53:01,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/c950497ebd1845b49f68062199808701 is 50, key is test_row_0/B:col10/1731358381048/Put/seqid=0 2024-11-11T20:53:01,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741895_1071 (size=12301) 2024-11-11T20:53:01,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T20:53:01,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:01,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:53:01,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:01,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:01,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:01,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T20:53:01,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T20:53:01,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:01,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:53:01,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:01,776 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:01,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:01,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:01,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:01,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358441776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:01,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:01,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:01,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:01,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:01,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358441776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:01,781 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:01,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:01,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358441846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:01,851 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_0' in 09438051ac0c3c1f634095c776d91044 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:01,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:01,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:01,852 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:01,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:01,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358441857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:01,859 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in 09438051ac0c3c1f634095c776d91044 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4220 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:01,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:01,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:01,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:01,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:01,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358441866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:01,877 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_1' in 09438051ac0c3c1f634095c776d91044 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4244 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:01,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:01,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:01,877 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:01,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T20:53:01,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:01,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:53:01,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:01,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:01,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:01,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:01,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/c950497ebd1845b49f68062199808701 2024-11-11T20:53:01,983 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/b6ac927f9cf744498fd1e9c800caa6cb is 50, key is test_row_0/C:col10/1731358381048/Put/seqid=0 2024-11-11T20:53:02,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741896_1072 (size=12301) 2024-11-11T20:53:02,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T20:53:02,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:02,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:53:02,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:02,085 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:02,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T20:53:02,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T20:53:02,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:02,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:53:02,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:02,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:02,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:02,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:02,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:02,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358442286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:02,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:02,290 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:02,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:02,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:02,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358442296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:02,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:02,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:02,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T20:53:02,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:02,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:53:02,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:02,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:02,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:02,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:02,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=563 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/b6ac927f9cf744498fd1e9c800caa6cb 2024-11-11T20:53:02,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/f2c88d22774f434f9712e0e25573ce83 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/f2c88d22774f434f9712e0e25573ce83 2024-11-11T20:53:02,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/f2c88d22774f434f9712e0e25573ce83, entries=200, sequenceid=563, filesize=14.4 K 2024-11-11T20:53:02,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/c950497ebd1845b49f68062199808701 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/c950497ebd1845b49f68062199808701 2024-11-11T20:53:02,466 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/c950497ebd1845b49f68062199808701, entries=150, sequenceid=563, filesize=12.0 K 2024-11-11T20:53:02,468 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/b6ac927f9cf744498fd1e9c800caa6cb as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/b6ac927f9cf744498fd1e9c800caa6cb 2024-11-11T20:53:02,482 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/b6ac927f9cf744498fd1e9c800caa6cb, entries=150, sequenceid=563, filesize=12.0 K 2024-11-11T20:53:02,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 09438051ac0c3c1f634095c776d91044 in 1434ms, sequenceid=563, compaction requested=true 2024-11-11T20:53:02,484 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:53:02,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09438051ac0c3c1f634095c776d91044:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:53:02,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:02,484 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:02,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09438051ac0c3c1f634095c776d91044:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:53:02,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:02,485 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:02,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09438051ac0c3c1f634095c776d91044:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:53:02,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:02,486 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:02,486 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 09438051ac0c3c1f634095c776d91044/A is initiating minor compaction (all files) 2024-11-11T20:53:02,486 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09438051ac0c3c1f634095c776d91044/A in TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:02,486 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/09f55155819f42389b23feb2350f39b9, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/4538847c1c4a4a5e96725747ca79cd8b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/f2c88d22774f434f9712e0e25573ce83] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp, totalSize=41.4 K 2024-11-11T20:53:02,487 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:02,487 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 09438051ac0c3c1f634095c776d91044/B is initiating minor compaction (all files) 2024-11-11T20:53:02,487 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09438051ac0c3c1f634095c776d91044/B in TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:02,487 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/a5000c108f1449cc883ff3fdc0604a21, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/176c60c329514f86b84ae5094e96cb12, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/c950497ebd1845b49f68062199808701] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp, totalSize=36.6 K 2024-11-11T20:53:02,487 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09f55155819f42389b23feb2350f39b9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1731358378815 2024-11-11T20:53:02,488 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting a5000c108f1449cc883ff3fdc0604a21, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1731358378815 2024-11-11T20:53:02,490 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4538847c1c4a4a5e96725747ca79cd8b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=551, earliestPutTs=1731358379735 2024-11-11T20:53:02,491 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 176c60c329514f86b84ae5094e96cb12, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=551, earliestPutTs=1731358379735 2024-11-11T20:53:02,492 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2c88d22774f434f9712e0e25573ce83, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1731358380898 2024-11-11T20:53:02,493 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting c950497ebd1845b49f68062199808701, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1731358380929 2024-11-11T20:53:02,524 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09438051ac0c3c1f634095c776d91044#A#compaction#60 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:02,524 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/bc8d904f71eb43f2a1556ddd9a74619d is 50, key is test_row_0/A:col10/1731358381048/Put/seqid=0 2024-11-11T20:53:02,531 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09438051ac0c3c1f634095c776d91044#B#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:02,532 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/0bc19ea98d9841efba5587a097e75304 is 50, key is test_row_0/B:col10/1731358381048/Put/seqid=0 2024-11-11T20:53:02,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741897_1073 (size=12983) 2024-11-11T20:53:02,549 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T20:53:02,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:02,550 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 09438051ac0c3c1f634095c776d91044 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T20:53:02,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=A 2024-11-11T20:53:02,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:02,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=B 2024-11-11T20:53:02,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:02,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=C 2024-11-11T20:53:02,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:02,564 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/bc8d904f71eb43f2a1556ddd9a74619d as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/bc8d904f71eb43f2a1556ddd9a74619d 2024-11-11T20:53:02,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/68b598035a24447b8abfdd727156ac1a is 50, key is test_row_0/A:col10/1731358381145/Put/seqid=0 2024-11-11T20:53:02,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741898_1074 (size=12983) 2024-11-11T20:53:02,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741899_1075 (size=12301) 2024-11-11T20:53:02,577 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=583 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/68b598035a24447b8abfdd727156ac1a 2024-11-11T20:53:02,588 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09438051ac0c3c1f634095c776d91044/A of 09438051ac0c3c1f634095c776d91044 into bc8d904f71eb43f2a1556ddd9a74619d(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:02,588 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:53:02,588 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., storeName=09438051ac0c3c1f634095c776d91044/A, priority=13, startTime=1731358382484; duration=0sec 2024-11-11T20:53:02,589 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:02,589 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09438051ac0c3c1f634095c776d91044:A 2024-11-11T20:53:02,589 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:02,605 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:02,606 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 09438051ac0c3c1f634095c776d91044/C is initiating minor compaction (all files) 2024-11-11T20:53:02,606 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09438051ac0c3c1f634095c776d91044/C in TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:02,606 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/0bc19ea98d9841efba5587a097e75304 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/0bc19ea98d9841efba5587a097e75304 2024-11-11T20:53:02,606 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/46d5c775659a46d0bd9ae1d3cd2e6072, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/d5d81f4a269b4ad480df2d031f7d73f6, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/b6ac927f9cf744498fd1e9c800caa6cb] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp, totalSize=36.6 K 2024-11-11T20:53:02,606 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46d5c775659a46d0bd9ae1d3cd2e6072, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=511, earliestPutTs=1731358378815 2024-11-11T20:53:02,607 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting d5d81f4a269b4ad480df2d031f7d73f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=551, earliestPutTs=1731358379735 2024-11-11T20:53:02,607 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6ac927f9cf744498fd1e9c800caa6cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1731358380929 2024-11-11T20:53:02,620 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09438051ac0c3c1f634095c776d91044/B of 09438051ac0c3c1f634095c776d91044 into 0bc19ea98d9841efba5587a097e75304(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:02,620 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:53:02,620 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., storeName=09438051ac0c3c1f634095c776d91044/B, priority=13, startTime=1731358382485; duration=0sec 2024-11-11T20:53:02,620 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:02,620 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09438051ac0c3c1f634095c776d91044:B 2024-11-11T20:53:02,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/96985a4ce41548aabb15dc580f3f3fc0 is 50, key is test_row_0/B:col10/1731358381145/Put/seqid=0 2024-11-11T20:53:02,626 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09438051ac0c3c1f634095c776d91044#C#compaction#63 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:02,627 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/2eb88bb4726b4d4ea59bafdd2f56ccca is 50, key is test_row_0/C:col10/1731358381048/Put/seqid=0 2024-11-11T20:53:02,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741900_1076 (size=12983) 2024-11-11T20:53:02,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741901_1077 (size=12301) 2024-11-11T20:53:02,666 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=583 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/96985a4ce41548aabb15dc580f3f3fc0 2024-11-11T20:53:02,666 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/2eb88bb4726b4d4ea59bafdd2f56ccca as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/2eb88bb4726b4d4ea59bafdd2f56ccca 2024-11-11T20:53:02,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/58efceee1c8c40839121d9dbb7894396 is 50, key is test_row_0/C:col10/1731358381145/Put/seqid=0 2024-11-11T20:53:02,694 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09438051ac0c3c1f634095c776d91044/C of 09438051ac0c3c1f634095c776d91044 into 2eb88bb4726b4d4ea59bafdd2f56ccca(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:02,694 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:53:02,694 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., storeName=09438051ac0c3c1f634095c776d91044/C, priority=13, startTime=1731358382485; duration=0sec 2024-11-11T20:53:02,695 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:02,695 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09438051ac0c3c1f634095c776d91044:C 2024-11-11T20:53:02,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741902_1078 (size=12301) 2024-11-11T20:53:03,101 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=583 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/58efceee1c8c40839121d9dbb7894396 2024-11-11T20:53:03,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/68b598035a24447b8abfdd727156ac1a as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/68b598035a24447b8abfdd727156ac1a 2024-11-11T20:53:03,121 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/68b598035a24447b8abfdd727156ac1a, entries=150, sequenceid=583, filesize=12.0 K 2024-11-11T20:53:03,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/96985a4ce41548aabb15dc580f3f3fc0 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/96985a4ce41548aabb15dc580f3f3fc0 2024-11-11T20:53:03,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T20:53:03,146 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/96985a4ce41548aabb15dc580f3f3fc0, entries=150, sequenceid=583, filesize=12.0 K 2024-11-11T20:53:03,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/58efceee1c8c40839121d9dbb7894396 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/58efceee1c8c40839121d9dbb7894396 2024-11-11T20:53:03,161 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/58efceee1c8c40839121d9dbb7894396, entries=150, sequenceid=583, filesize=12.0 K 2024-11-11T20:53:03,163 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 09438051ac0c3c1f634095c776d91044 in 613ms, sequenceid=583, compaction requested=false 2024-11-11T20:53:03,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:53:03,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:03,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-11T20:53:03,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-11T20:53:03,168 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-11T20:53:03,169 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1640 sec 2024-11-11T20:53:03,175 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.1760 sec 2024-11-11T20:53:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 09438051ac0c3c1f634095c776d91044 2024-11-11T20:53:03,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09438051ac0c3c1f634095c776d91044 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T20:53:03,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=A 2024-11-11T20:53:03,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:03,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=B 2024-11-11T20:53:03,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:03,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=C 2024-11-11T20:53:03,326 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:03,343 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/df02e313a1d74cd9a3efc91a0fcf0a19 is 50, key is test_row_0/A:col10/1731358383321/Put/seqid=0 2024-11-11T20:53:03,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741903_1079 (size=17181) 2024-11-11T20:53:03,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358383386, max timestamp=1731358383402Num uniques 859; 2024-11-11T20:53:03,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731358383368, max timestamp=1731358383402Num uniques 859; 2024-11-11T20:53:03,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T20:53:03,406 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:53:03,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358383386, max timestamp=1731358383402Num uniques 859; 2024-11-11T20:53:03,409 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:53:03,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:03,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358443451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:03,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:03,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:03,457 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:03,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:03,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358443456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:03,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:03,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:03,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:03,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 195 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358443566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:03,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:03,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358443567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:03,571 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:03,572 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:03,755 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=597 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/df02e313a1d74cd9a3efc91a0fcf0a19 2024-11-11T20:53:03,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/79dde07d2ea94a91928c89bea285f2e3 is 50, key is test_row_0/B:col10/1731358383321/Put/seqid=0 2024-11-11T20:53:03,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741904_1080 (size=12301) 2024-11-11T20:53:03,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:03,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 196 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358443776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:03,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:03,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:03,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358443776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:03,779 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:04,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:04,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 197 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358444086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:04,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:04,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:04,092 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:04,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:04,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358444087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:04,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:04,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:04,097 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:04,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=597 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/79dde07d2ea94a91928c89bea285f2e3 2024-11-11T20:53:04,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/53cee8e1caeb42cd9281b6eb2b06bb16 is 50, key is test_row_0/C:col10/1731358383321/Put/seqid=0 2024-11-11T20:53:04,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741905_1081 (size=12301) 2024-11-11T20:53:04,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=597 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/53cee8e1caeb42cd9281b6eb2b06bb16 2024-11-11T20:53:04,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/df02e313a1d74cd9a3efc91a0fcf0a19 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/df02e313a1d74cd9a3efc91a0fcf0a19 2024-11-11T20:53:04,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:04,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358444606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:04,612 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:04,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358444607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:04,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:04,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:04,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:04,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:04,613 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:04,613 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:04,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/df02e313a1d74cd9a3efc91a0fcf0a19, entries=250, sequenceid=597, filesize=16.8 K 2024-11-11T20:53:04,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/79dde07d2ea94a91928c89bea285f2e3 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/79dde07d2ea94a91928c89bea285f2e3 2024-11-11T20:53:04,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/79dde07d2ea94a91928c89bea285f2e3, entries=150, sequenceid=597, filesize=12.0 K 2024-11-11T20:53:04,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/53cee8e1caeb42cd9281b6eb2b06bb16 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/53cee8e1caeb42cd9281b6eb2b06bb16 2024-11-11T20:53:04,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/53cee8e1caeb42cd9281b6eb2b06bb16, entries=150, sequenceid=597, filesize=12.0 K 2024-11-11T20:53:04,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=125.23 KB/128240 for 09438051ac0c3c1f634095c776d91044 in 1312ms, sequenceid=597, compaction requested=true 2024-11-11T20:53:04,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:53:04,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09438051ac0c3c1f634095c776d91044:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:53:04,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:04,637 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:04,637 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09438051ac0c3c1f634095c776d91044:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:53:04,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:04,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09438051ac0c3c1f634095c776d91044:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:53:04,638 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:04,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:04,640 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:04,640 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 09438051ac0c3c1f634095c776d91044/B is initiating minor compaction (all files) 2024-11-11T20:53:04,640 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09438051ac0c3c1f634095c776d91044/B in TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:04,640 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/0bc19ea98d9841efba5587a097e75304, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/96985a4ce41548aabb15dc580f3f3fc0, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/79dde07d2ea94a91928c89bea285f2e3] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp, totalSize=36.7 K 2024-11-11T20:53:04,641 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42465 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:04,641 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 0bc19ea98d9841efba5587a097e75304, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1731358380929 2024-11-11T20:53:04,641 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 09438051ac0c3c1f634095c776d91044/A is initiating minor compaction (all files) 2024-11-11T20:53:04,641 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09438051ac0c3c1f634095c776d91044/A in TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:04,641 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/bc8d904f71eb43f2a1556ddd9a74619d, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/68b598035a24447b8abfdd727156ac1a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/df02e313a1d74cd9a3efc91a0fcf0a19] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp, totalSize=41.5 K 2024-11-11T20:53:04,642 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 96985a4ce41548aabb15dc580f3f3fc0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=583, earliestPutTs=1731358381122 2024-11-11T20:53:04,642 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting bc8d904f71eb43f2a1556ddd9a74619d, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1731358380929 2024-11-11T20:53:04,643 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 68b598035a24447b8abfdd727156ac1a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=583, earliestPutTs=1731358381122 2024-11-11T20:53:04,643 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting df02e313a1d74cd9a3efc91a0fcf0a19, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=597, earliestPutTs=1731358383305 2024-11-11T20:53:04,646 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 79dde07d2ea94a91928c89bea285f2e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=597, earliestPutTs=1731358383308 2024-11-11T20:53:04,657 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09438051ac0c3c1f634095c776d91044#B#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:04,658 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/90afc8255fa14fe5a8d5f873ebef0eda is 50, key is test_row_0/B:col10/1731358383321/Put/seqid=0 2024-11-11T20:53:04,663 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09438051ac0c3c1f634095c776d91044#A#compaction#70 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:04,664 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/8d135ae066f74003b083ddacf36d881b is 50, key is test_row_0/A:col10/1731358383321/Put/seqid=0 2024-11-11T20:53:04,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741907_1083 (size=13085) 2024-11-11T20:53:04,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741906_1082 (size=13085) 2024-11-11T20:53:04,697 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/8d135ae066f74003b083ddacf36d881b as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/8d135ae066f74003b083ddacf36d881b 2024-11-11T20:53:04,706 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09438051ac0c3c1f634095c776d91044/A of 09438051ac0c3c1f634095c776d91044 into 8d135ae066f74003b083ddacf36d881b(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:04,706 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:53:04,706 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., storeName=09438051ac0c3c1f634095c776d91044/A, priority=13, startTime=1731358384637; duration=0sec 2024-11-11T20:53:04,706 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:04,706 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09438051ac0c3c1f634095c776d91044:A 2024-11-11T20:53:04,706 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:04,708 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:04,708 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 09438051ac0c3c1f634095c776d91044/C is initiating minor compaction (all files) 2024-11-11T20:53:04,708 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09438051ac0c3c1f634095c776d91044/C in TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:04,708 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/2eb88bb4726b4d4ea59bafdd2f56ccca, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/58efceee1c8c40839121d9dbb7894396, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/53cee8e1caeb42cd9281b6eb2b06bb16] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp, totalSize=36.7 K 2024-11-11T20:53:04,710 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2eb88bb4726b4d4ea59bafdd2f56ccca, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=563, earliestPutTs=1731358380929 2024-11-11T20:53:04,710 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58efceee1c8c40839121d9dbb7894396, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=583, earliestPutTs=1731358381122 2024-11-11T20:53:04,710 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53cee8e1caeb42cd9281b6eb2b06bb16, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=597, earliestPutTs=1731358383308 2024-11-11T20:53:04,720 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09438051ac0c3c1f634095c776d91044#C#compaction#71 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:04,721 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/c9c9d582bc7b4ce9bde0f85c24bf9079 is 50, key is test_row_0/C:col10/1731358383321/Put/seqid=0 2024-11-11T20:53:04,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741908_1084 (size=13085) 2024-11-11T20:53:05,101 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/90afc8255fa14fe5a8d5f873ebef0eda as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/90afc8255fa14fe5a8d5f873ebef0eda 2024-11-11T20:53:05,109 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09438051ac0c3c1f634095c776d91044/B of 09438051ac0c3c1f634095c776d91044 into 90afc8255fa14fe5a8d5f873ebef0eda(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:05,109 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:53:05,109 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., storeName=09438051ac0c3c1f634095c776d91044/B, priority=13, startTime=1731358384637; duration=0sec 2024-11-11T20:53:05,110 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:05,110 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09438051ac0c3c1f634095c776d91044:B 2024-11-11T20:53:05,138 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/c9c9d582bc7b4ce9bde0f85c24bf9079 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/c9c9d582bc7b4ce9bde0f85c24bf9079 2024-11-11T20:53:05,147 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09438051ac0c3c1f634095c776d91044/C of 09438051ac0c3c1f634095c776d91044 into c9c9d582bc7b4ce9bde0f85c24bf9079(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:05,148 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:53:05,148 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., storeName=09438051ac0c3c1f634095c776d91044/C, priority=13, startTime=1731358384638; duration=0sec 2024-11-11T20:53:05,148 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:05,148 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09438051ac0c3c1f634095c776d91044:C 2024-11-11T20:53:05,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T20:53:05,176 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:53:05,176 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:53:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-11T20:53:05,179 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:53:05,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T20:53:05,180 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:53:05,180 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:53:05,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T20:53:05,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T20:53:05,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:05,332 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 09438051ac0c3c1f634095c776d91044 3/3 column families, dataSize=125.23 KB heapSize=355.30 KB 2024-11-11T20:53:05,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=A 2024-11-11T20:53:05,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:05,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=B 2024-11-11T20:53:05,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:05,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=C 2024-11-11T20:53:05,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:05,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/185c7fa2bcdb48c7aa72a6dd2505f44a is 50, key is test_row_0/A:col10/1731358383446/Put/seqid=0 2024-11-11T20:53:05,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741909_1085 (size=12301) 2024-11-11T20:53:05,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T20:53:05,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:53:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 09438051ac0c3c1f634095c776d91044 2024-11-11T20:53:05,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358445653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:05,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:05,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358445653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:05,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:05,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:05,743 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/185c7fa2bcdb48c7aa72a6dd2505f44a 2024-11-11T20:53:05,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/151fc54ef83b4083b9df93d921b7de8c is 50, key is test_row_0/B:col10/1731358383446/Put/seqid=0 2024-11-11T20:53:05,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741910_1086 (size=12301) 2024-11-11T20:53:05,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:05,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358445776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:05,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:05,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:05,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:05,781 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/151fc54ef83b4083b9df93d921b7de8c 2024-11-11T20:53:05,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:05,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358445776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:05,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:05,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:05,782 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:05,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/e8843c44fea445f9816b44636d741cbc is 50, key is test_row_0/C:col10/1731358383446/Put/seqid=0 2024-11-11T20:53:05,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T20:53:05,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741911_1087 (size=12301) 2024-11-11T20:53:05,815 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/e8843c44fea445f9816b44636d741cbc 2024-11-11T20:53:05,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/185c7fa2bcdb48c7aa72a6dd2505f44a as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/185c7fa2bcdb48c7aa72a6dd2505f44a 2024-11-11T20:53:05,837 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/185c7fa2bcdb48c7aa72a6dd2505f44a, entries=150, sequenceid=635, filesize=12.0 K 2024-11-11T20:53:05,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/151fc54ef83b4083b9df93d921b7de8c as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/151fc54ef83b4083b9df93d921b7de8c 2024-11-11T20:53:05,849 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/151fc54ef83b4083b9df93d921b7de8c, entries=150, sequenceid=635, filesize=12.0 K 2024-11-11T20:53:05,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/e8843c44fea445f9816b44636d741cbc as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/e8843c44fea445f9816b44636d741cbc 2024-11-11T20:53:05,857 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/e8843c44fea445f9816b44636d741cbc, entries=150, sequenceid=635, filesize=12.0 K 2024-11-11T20:53:05,858 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~125.23 KB/128240, heapSize ~355.25 KB/363776, currentSize=60.38 KB/61830 for 09438051ac0c3c1f634095c776d91044 in 526ms, sequenceid=635, compaction requested=false 2024-11-11T20:53:05,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:53:05,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:05,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-11T20:53:05,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-11T20:53:05,866 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-11T20:53:05,866 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 681 msec 2024-11-11T20:53:05,869 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 691 msec 2024-11-11T20:53:05,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 09438051ac0c3c1f634095c776d91044 2024-11-11T20:53:05,888 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09438051ac0c3c1f634095c776d91044 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T20:53:05,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=A 2024-11-11T20:53:05,888 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:05,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=B 2024-11-11T20:53:05,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:05,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=C 2024-11-11T20:53:05,889 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:05,897 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/6f9c3780567f40b08da4c3aebaf33813 is 50, key is test_row_0/A:col10/1731358385887/Put/seqid=0 2024-11-11T20:53:05,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741912_1088 (size=12301) 2024-11-11T20:53:05,930 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=648 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/6f9c3780567f40b08da4c3aebaf33813 2024-11-11T20:53:05,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:05,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358445925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:05,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:05,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:05,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358445928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:05,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:05,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:05,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:05,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:05,932 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:05,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358445931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:05,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:05,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:05,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:05,941 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/87c7062f7f6c4bb2bda9624071da8386 is 50, key is test_row_0/B:col10/1731358385887/Put/seqid=0 2024-11-11T20:53:05,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741913_1089 (size=12301) 2024-11-11T20:53:05,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:05,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358445986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:05,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:05,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:05,987 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:05,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:05,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 193 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358445989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:05,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:05,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:05,992 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:06,037 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358446036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:06,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:06,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:06,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:06,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:06,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358446036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:06,039 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:06,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:06,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358446046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:06,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:06,048 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:06,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:06,254 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358446246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:06,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:06,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:06,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:06,256 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:06,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358446246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:06,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:06,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:06,256 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:06,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:06,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358446256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:06,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:06,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:06,260 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:06,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358446296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:06,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:06,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:06,297 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:06,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:06,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 194 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358446306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:06,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:06,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:06,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:06,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T20:53:06,316 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:53:06,316 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:53:06,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-11T20:53:06,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T20:53:06,318 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:53:06,319 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:53:06,319 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:53:06,353 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=648 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/87c7062f7f6c4bb2bda9624071da8386 2024-11-11T20:53:06,364 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/9489053f5d544dd0a1d3b18b2b8d977f is 50, key is test_row_0/C:col10/1731358385887/Put/seqid=0 2024-11-11T20:53:06,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741914_1090 (size=12301) 2024-11-11T20:53:06,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T20:53:06,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T20:53:06,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:06,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:53:06,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:06,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:06,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:06,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:06,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358446565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:06,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:06,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358446565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:06,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:06,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:06,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:06,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:06,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358446565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:06,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:06,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:06,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:06,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:06,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:06,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:06,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T20:53:06,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:06,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:53:06,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:06,624 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:06,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:06,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T20:53:06,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T20:53:06,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:06,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:53:06,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:06,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:06,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:06,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:06,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=648 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/9489053f5d544dd0a1d3b18b2b8d977f 2024-11-11T20:53:06,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/6f9c3780567f40b08da4c3aebaf33813 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/6f9c3780567f40b08da4c3aebaf33813 2024-11-11T20:53:06,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/6f9c3780567f40b08da4c3aebaf33813, entries=150, sequenceid=648, filesize=12.0 K 2024-11-11T20:53:06,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/87c7062f7f6c4bb2bda9624071da8386 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/87c7062f7f6c4bb2bda9624071da8386 2024-11-11T20:53:06,804 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/87c7062f7f6c4bb2bda9624071da8386, entries=150, sequenceid=648, filesize=12.0 K 2024-11-11T20:53:06,805 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/9489053f5d544dd0a1d3b18b2b8d977f as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/9489053f5d544dd0a1d3b18b2b8d977f 2024-11-11T20:53:06,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:06,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358446806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:06,807 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:06,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/9489053f5d544dd0a1d3b18b2b8d977f, entries=150, sequenceid=648, filesize=12.0 K 2024-11-11T20:53:06,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 09438051ac0c3c1f634095c776d91044 in 923ms, sequenceid=648, compaction requested=true 2024-11-11T20:53:06,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:53:06,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09438051ac0c3c1f634095c776d91044:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:53:06,812 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:06,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:06,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09438051ac0c3c1f634095c776d91044:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:53:06,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:06,812 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:06,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09438051ac0c3c1f634095c776d91044:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:53:06,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:06,813 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:06,813 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 09438051ac0c3c1f634095c776d91044/A is initiating minor compaction (all files) 2024-11-11T20:53:06,813 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:06,813 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09438051ac0c3c1f634095c776d91044/A in TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:06,813 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 09438051ac0c3c1f634095c776d91044/B is initiating minor compaction (all files) 2024-11-11T20:53:06,813 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09438051ac0c3c1f634095c776d91044/B in TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:06,813 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/8d135ae066f74003b083ddacf36d881b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/185c7fa2bcdb48c7aa72a6dd2505f44a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/6f9c3780567f40b08da4c3aebaf33813] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp, totalSize=36.8 K 2024-11-11T20:53:06,813 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/90afc8255fa14fe5a8d5f873ebef0eda, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/151fc54ef83b4083b9df93d921b7de8c, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/87c7062f7f6c4bb2bda9624071da8386] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp, totalSize=36.8 K 2024-11-11T20:53:06,814 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d135ae066f74003b083ddacf36d881b, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=597, earliestPutTs=1731358383308 2024-11-11T20:53:06,814 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 90afc8255fa14fe5a8d5f873ebef0eda, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=597, earliestPutTs=1731358383308 2024-11-11T20:53:06,814 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 185c7fa2bcdb48c7aa72a6dd2505f44a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=635, earliestPutTs=1731358383446 2024-11-11T20:53:06,814 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 151fc54ef83b4083b9df93d921b7de8c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=635, earliestPutTs=1731358383446 2024-11-11T20:53:06,814 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f9c3780567f40b08da4c3aebaf33813, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=648, earliestPutTs=1731358385627 2024-11-11T20:53:06,815 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 87c7062f7f6c4bb2bda9624071da8386, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=648, earliestPutTs=1731358385627 2024-11-11T20:53:06,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 09438051ac0c3c1f634095c776d91044 2024-11-11T20:53:06,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09438051ac0c3c1f634095c776d91044 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T20:53:06,824 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09438051ac0c3c1f634095c776d91044#A#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:06,824 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/c3f035f73cfe462fadc87e0b7ddf91c2 is 50, key is test_row_0/A:col10/1731358385887/Put/seqid=0 2024-11-11T20:53:06,824 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09438051ac0c3c1f634095c776d91044#B#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:06,825 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/45d7d8cdcfc74d18ba71464f2db2c621 is 50, key is test_row_0/B:col10/1731358385887/Put/seqid=0 2024-11-11T20:53:06,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=A 2024-11-11T20:53:06,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:06,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=B 2024-11-11T20:53:06,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:06,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=C 2024-11-11T20:53:06,826 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:06,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/14eae057530f4703ae38162a40d6a308 is 50, key is test_row_0/A:col10/1731358386816/Put/seqid=0 2024-11-11T20:53:06,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741915_1091 (size=13187) 2024-11-11T20:53:06,854 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/c3f035f73cfe462fadc87e0b7ddf91c2 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/c3f035f73cfe462fadc87e0b7ddf91c2 2024-11-11T20:53:06,861 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09438051ac0c3c1f634095c776d91044/A of 09438051ac0c3c1f634095c776d91044 into c3f035f73cfe462fadc87e0b7ddf91c2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:06,861 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:53:06,861 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., storeName=09438051ac0c3c1f634095c776d91044/A, priority=13, startTime=1731358386812; duration=0sec 2024-11-11T20:53:06,861 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:06,861 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09438051ac0c3c1f634095c776d91044:A 2024-11-11T20:53:06,861 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:06,863 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:06,863 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 09438051ac0c3c1f634095c776d91044/C is initiating minor compaction (all files) 2024-11-11T20:53:06,863 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09438051ac0c3c1f634095c776d91044/C in TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:06,864 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/c9c9d582bc7b4ce9bde0f85c24bf9079, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/e8843c44fea445f9816b44636d741cbc, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/9489053f5d544dd0a1d3b18b2b8d977f] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp, totalSize=36.8 K 2024-11-11T20:53:06,864 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting c9c9d582bc7b4ce9bde0f85c24bf9079, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=597, earliestPutTs=1731358383308 2024-11-11T20:53:06,864 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8843c44fea445f9816b44636d741cbc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=635, earliestPutTs=1731358383446 2024-11-11T20:53:06,865 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9489053f5d544dd0a1d3b18b2b8d977f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=648, earliestPutTs=1731358385627 2024-11-11T20:53:06,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741916_1092 (size=13187) 2024-11-11T20:53:06,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741917_1093 (size=14741) 2024-11-11T20:53:06,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=669 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/14eae057530f4703ae38162a40d6a308 2024-11-11T20:53:06,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:06,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358446872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:06,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:06,887 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09438051ac0c3c1f634095c776d91044#C#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:06,888 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/697336280ee4441dbe4648b1d978ffc9 is 50, key is test_row_0/C:col10/1731358385887/Put/seqid=0 2024-11-11T20:53:06,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/9bd4bd4faca84bf892d6f883bc2f9112 is 50, key is test_row_0/B:col10/1731358386816/Put/seqid=0 2024-11-11T20:53:06,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741918_1094 (size=12301) 2024-11-11T20:53:06,901 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=669 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/9bd4bd4faca84bf892d6f883bc2f9112 2024-11-11T20:53:06,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/e5641ad004b14b91b248d9ad6dedc492 is 50, key is test_row_0/C:col10/1731358386816/Put/seqid=0 2024-11-11T20:53:06,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741919_1095 (size=13187) 2024-11-11T20:53:06,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T20:53:06,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:06,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:53:06,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:06,930 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:06,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:06,931 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/697336280ee4441dbe4648b1d978ffc9 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/697336280ee4441dbe4648b1d978ffc9 2024-11-11T20:53:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:06,943 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09438051ac0c3c1f634095c776d91044/C of 09438051ac0c3c1f634095c776d91044 into 697336280ee4441dbe4648b1d978ffc9(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:06,945 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:53:06,945 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., storeName=09438051ac0c3c1f634095c776d91044/C, priority=13, startTime=1731358386812; duration=0sec 2024-11-11T20:53:06,945 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:06,945 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09438051ac0c3c1f634095c776d91044:C 2024-11-11T20:53:06,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T20:53:06,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741920_1096 (size=12301) 2024-11-11T20:53:06,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:06,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 206 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358446985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:06,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:06,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:06,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:07,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358447075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:07,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:07,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:07,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:07,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358447076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:07,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:07,078 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:07,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:07,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T20:53:07,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:07,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:53:07,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:07,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:07,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:07,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:07,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:07,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358447086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:07,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:07,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:07,087 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:07,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:07,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 207 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358447195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:07,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:07,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T20:53:07,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:07,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:53:07,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:07,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:07,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:07,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:07,283 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/45d7d8cdcfc74d18ba71464f2db2c621 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/45d7d8cdcfc74d18ba71464f2db2c621 2024-11-11T20:53:07,289 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 09438051ac0c3c1f634095c776d91044/B of 09438051ac0c3c1f634095c776d91044 into 45d7d8cdcfc74d18ba71464f2db2c621(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:07,289 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:53:07,289 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., storeName=09438051ac0c3c1f634095c776d91044/B, priority=13, startTime=1731358386812; duration=0sec 2024-11-11T20:53:07,289 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:07,289 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09438051ac0c3c1f634095c776d91044:B 2024-11-11T20:53:07,348 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=669 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/e5641ad004b14b91b248d9ad6dedc492 2024-11-11T20:53:07,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/14eae057530f4703ae38162a40d6a308 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/14eae057530f4703ae38162a40d6a308 2024-11-11T20:53:07,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/14eae057530f4703ae38162a40d6a308, entries=200, sequenceid=669, filesize=14.4 K 2024-11-11T20:53:07,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/9bd4bd4faca84bf892d6f883bc2f9112 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/9bd4bd4faca84bf892d6f883bc2f9112 2024-11-11T20:53:07,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/9bd4bd4faca84bf892d6f883bc2f9112, entries=150, sequenceid=669, filesize=12.0 K 2024-11-11T20:53:07,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/e5641ad004b14b91b248d9ad6dedc492 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/e5641ad004b14b91b248d9ad6dedc492 2024-11-11T20:53:07,373 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/e5641ad004b14b91b248d9ad6dedc492, entries=150, sequenceid=669, filesize=12.0 K 2024-11-11T20:53:07,374 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 09438051ac0c3c1f634095c776d91044 in 555ms, sequenceid=669, compaction requested=false 2024-11-11T20:53:07,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:53:07,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T20:53:07,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:07,390 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 09438051ac0c3c1f634095c776d91044 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T20:53:07,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=A 2024-11-11T20:53:07,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:07,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=B 2024-11-11T20:53:07,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:07,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=C 2024-11-11T20:53:07,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:07,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/002381ece1d24f33bc246529d646bbde is 50, key is test_row_0/A:col10/1731358386842/Put/seqid=0 2024-11-11T20:53:07,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741921_1097 (size=12301) 2024-11-11T20:53:07,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T20:53:07,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 09438051ac0c3c1f634095c776d91044 2024-11-11T20:53:07,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. as already flushing 2024-11-11T20:53:07,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358387539, max timestamp=1731358387561Num uniques 859; 2024-11-11T20:53:07,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:53:07,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358387539, max timestamp=1731358387561Num uniques 859; 2024-11-11T20:53:07,564 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:53:07,613 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:07,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 235 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358447608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:07,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:07,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:07,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:07,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:07,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 236 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358447715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:07,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:07,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:07,719 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:07,805 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=683 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/002381ece1d24f33bc246529d646bbde 2024-11-11T20:53:07,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/e43a7574361b4fe5b3425290ce3255b3 is 50, key is test_row_0/B:col10/1731358386842/Put/seqid=0 2024-11-11T20:53:07,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:07,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49338 deadline: 1731358447825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:07,830 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_0' in 09438051ac0c3c1f634095c776d91044 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:07,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741922_1098 (size=12301) 2024-11-11T20:53:07,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:07,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 237 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358447925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:07,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:07,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:07,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:08,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:08,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49304 deadline: 1731358448086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:08,094 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_0' in 09438051ac0c3c1f634095c776d91044 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:08,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:08,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:08,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:08,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:08,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49288 deadline: 1731358448096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:08,097 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:08,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49278 deadline: 1731358448097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:08,097 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in 09438051ac0c3c1f634095c776d91044 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:08,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:08,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:08,098 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:08,097 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_0' in 09438051ac0c3c1f634095c776d91044 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:08,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:08,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:08,098 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:08,232 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=683 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/e43a7574361b4fe5b3425290ce3255b3 2024-11-11T20:53:08,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/6666a37e15c3450f96b81fb3d58bc7c5 is 50, key is test_row_0/C:col10/1731358386842/Put/seqid=0 2024-11-11T20:53:08,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741923_1099 (size=12301) 2024-11-11T20:53:08,252 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:08,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 238 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49294 deadline: 1731358448246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:08,253 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:08,435 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-11T20:53:08,435 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:53:08,437 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-11T20:53:08,437 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:53:08,438 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-11T20:53:08,439 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:53:08,440 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:08,440 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:08,440 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:08,440 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-11T20:53:08,440 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:53:08,440 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:08,440 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:08,440 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:08,440 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:08,440 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:53:08,440 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:53:08,440 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:08,440 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:53:08,440 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:53:08,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T20:53:08,648 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=683 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/6666a37e15c3450f96b81fb3d58bc7c5 2024-11-11T20:53:08,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/002381ece1d24f33bc246529d646bbde as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/002381ece1d24f33bc246529d646bbde 2024-11-11T20:53:08,666 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/002381ece1d24f33bc246529d646bbde, entries=150, sequenceid=683, filesize=12.0 K 2024-11-11T20:53:08,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/e43a7574361b4fe5b3425290ce3255b3 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/e43a7574361b4fe5b3425290ce3255b3 2024-11-11T20:53:08,672 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/e43a7574361b4fe5b3425290ce3255b3, entries=150, sequenceid=683, filesize=12.0 K 2024-11-11T20:53:08,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/6666a37e15c3450f96b81fb3d58bc7c5 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/6666a37e15c3450f96b81fb3d58bc7c5 2024-11-11T20:53:08,678 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/6666a37e15c3450f96b81fb3d58bc7c5, entries=150, sequenceid=683, filesize=12.0 K 2024-11-11T20:53:08,679 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=123.00 KB/125950 for 09438051ac0c3c1f634095c776d91044 in 1290ms, sequenceid=683, compaction requested=true 2024-11-11T20:53:08,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:53:08,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:08,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-11T20:53:08,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-11T20:53:08,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-11T20:53:08,682 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3610 sec 2024-11-11T20:53:08,684 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 2.3670 sec 2024-11-11T20:53:08,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 09438051ac0c3c1f634095c776d91044 2024-11-11T20:53:08,768 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 09438051ac0c3c1f634095c776d91044 3/3 column families, dataSize=129.71 KB heapSize=373.55 KB 2024-11-11T20:53:08,768 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-11T20:53:08,768 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:53:08,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=A 2024-11-11T20:53:08,768 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:08,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:08,768 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:08,768 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:53:08,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=B 2024-11-11T20:53:08,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:08,768 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=C 2024-11-11T20:53:08,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:08,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/caeeff18b8fe42a1998a21108024bfb0 is 50, key is test_row_0/A:col10/1731358387573/Put/seqid=0 2024-11-11T20:53:08,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741924_1100 (size=12301) 2024-11-11T20:53:09,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/caeeff18b8fe42a1998a21108024bfb0 2024-11-11T20:53:09,199 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/8df5bb46bdff4962b61827b0089683e0 is 50, key is test_row_0/B:col10/1731358387573/Put/seqid=0 2024-11-11T20:53:09,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741925_1101 (size=12301) 2024-11-11T20:53:09,606 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/8df5bb46bdff4962b61827b0089683e0 2024-11-11T20:53:09,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/8ef334f8f7a04dabb0cc51f2e4498b78 is 50, key is test_row_0/C:col10/1731358387573/Put/seqid=0 2024-11-11T20:53:09,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741926_1102 (size=12301) 2024-11-11T20:53:09,841 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-11T20:53:09,841 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:53:09,841 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:09,841 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:09,841 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:53:10,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=714 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/8ef334f8f7a04dabb0cc51f2e4498b78 2024-11-11T20:53:10,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/caeeff18b8fe42a1998a21108024bfb0 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/caeeff18b8fe42a1998a21108024bfb0 2024-11-11T20:53:10,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/caeeff18b8fe42a1998a21108024bfb0, entries=150, sequenceid=714, filesize=12.0 K 2024-11-11T20:53:10,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/8df5bb46bdff4962b61827b0089683e0 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/8df5bb46bdff4962b61827b0089683e0 2024-11-11T20:53:10,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/8df5bb46bdff4962b61827b0089683e0, entries=150, sequenceid=714, filesize=12.0 K 2024-11-11T20:53:10,057 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/8ef334f8f7a04dabb0cc51f2e4498b78 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/8ef334f8f7a04dabb0cc51f2e4498b78 2024-11-11T20:53:10,062 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/8ef334f8f7a04dabb0cc51f2e4498b78, entries=150, sequenceid=714, filesize=12.0 K 2024-11-11T20:53:10,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~373.50 KB/382464, currentSize=6.71 KB/6870 for 09438051ac0c3c1f634095c776d91044 in 1296ms, sequenceid=714, compaction requested=true 2024-11-11T20:53:10,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:53:10,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09438051ac0c3c1f634095c776d91044:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:53:10,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:10,063 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:53:10,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09438051ac0c3c1f634095c776d91044:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:53:10,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:10,063 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:53:10,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 09438051ac0c3c1f634095c776d91044:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:53:10,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:10,065 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52530 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:53:10,065 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:53:10,065 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 09438051ac0c3c1f634095c776d91044/A is initiating minor compaction (all files) 2024-11-11T20:53:10,065 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 09438051ac0c3c1f634095c776d91044/B is initiating minor compaction (all files) 2024-11-11T20:53:10,065 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09438051ac0c3c1f634095c776d91044/A in TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:10,065 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09438051ac0c3c1f634095c776d91044/B in TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:10,065 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/c3f035f73cfe462fadc87e0b7ddf91c2, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/14eae057530f4703ae38162a40d6a308, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/002381ece1d24f33bc246529d646bbde, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/caeeff18b8fe42a1998a21108024bfb0] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp, totalSize=51.3 K 2024-11-11T20:53:10,065 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/45d7d8cdcfc74d18ba71464f2db2c621, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/9bd4bd4faca84bf892d6f883bc2f9112, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/e43a7574361b4fe5b3425290ce3255b3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/8df5bb46bdff4962b61827b0089683e0] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp, totalSize=48.9 K 2024-11-11T20:53:10,065 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 45d7d8cdcfc74d18ba71464f2db2c621, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=648, earliestPutTs=1731358385627 2024-11-11T20:53:10,065 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3f035f73cfe462fadc87e0b7ddf91c2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=648, earliestPutTs=1731358385627 2024-11-11T20:53:10,066 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bd4bd4faca84bf892d6f883bc2f9112, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=669, earliestPutTs=1731358385923 2024-11-11T20:53:10,066 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14eae057530f4703ae38162a40d6a308, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=669, earliestPutTs=1731358385918 2024-11-11T20:53:10,066 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting e43a7574361b4fe5b3425290ce3255b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=683, earliestPutTs=1731358386842 2024-11-11T20:53:10,066 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 002381ece1d24f33bc246529d646bbde, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=683, earliestPutTs=1731358386842 2024-11-11T20:53:10,066 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 8df5bb46bdff4962b61827b0089683e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=714, earliestPutTs=1731358387573 2024-11-11T20:53:10,066 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting caeeff18b8fe42a1998a21108024bfb0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=714, earliestPutTs=1731358387573 2024-11-11T20:53:10,075 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09438051ac0c3c1f634095c776d91044#A#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:10,076 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/5dc8a4d752f440d6a1aeafe47d1a8702 is 50, key is test_row_0/A:col10/1731358387573/Put/seqid=0 2024-11-11T20:53:10,076 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09438051ac0c3c1f634095c776d91044#B#compaction#91 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:10,077 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/ab665da0c8b0440b84281f0062fcc09b is 50, key is test_row_0/B:col10/1731358387573/Put/seqid=0 2024-11-11T20:53:10,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741927_1103 (size=13323) 2024-11-11T20:53:10,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741928_1104 (size=13323) 2024-11-11T20:53:10,107 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-11T20:53:10,107 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:53:10,107 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:10,107 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:10,107 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:53:10,117 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-11T20:53:10,118 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-11T20:53:10,118 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:53:10,118 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:10,118 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:53:10,118 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:10,118 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:10,118 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:10,118 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:53:10,118 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:53:10,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T20:53:10,477 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:53:10,477 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T20:53:10,478 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-11T20:53:10,478 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 157 2024-11-11T20:53:10,478 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-11-11T20:53:10,478 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 180 2024-11-11T20:53:10,478 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 90 2024-11-11T20:53:10,478 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T20:53:10,478 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2269 2024-11-11T20:53:10,478 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2217 2024-11-11T20:53:10,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T20:53:10,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1048 2024-11-11T20:53:10,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3138 rows 2024-11-11T20:53:10,479 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1047 2024-11-11T20:53:10,480 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3141 rows 2024-11-11T20:53:10,480 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T20:53:10,481 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:53:10,481 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:10,481 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:10,481 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:53:10,488 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T20:53:10,489 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/5dc8a4d752f440d6a1aeafe47d1a8702 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/5dc8a4d752f440d6a1aeafe47d1a8702 2024-11-11T20:53:10,489 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/ab665da0c8b0440b84281f0062fcc09b as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/ab665da0c8b0440b84281f0062fcc09b 2024-11-11T20:53:10,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T20:53:10,496 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 09438051ac0c3c1f634095c776d91044/B of 09438051ac0c3c1f634095c776d91044 into ab665da0c8b0440b84281f0062fcc09b(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:10,496 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 09438051ac0c3c1f634095c776d91044/A of 09438051ac0c3c1f634095c776d91044 into 5dc8a4d752f440d6a1aeafe47d1a8702(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:10,496 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:53:10,496 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:53:10,496 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., storeName=09438051ac0c3c1f634095c776d91044/A, priority=12, startTime=1731358390063; duration=0sec 2024-11-11T20:53:10,496 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., storeName=09438051ac0c3c1f634095c776d91044/B, priority=12, startTime=1731358390063; duration=0sec 2024-11-11T20:53:10,496 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:10,496 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:10,496 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09438051ac0c3c1f634095c776d91044:A 2024-11-11T20:53:10,496 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09438051ac0c3c1f634095c776d91044:B 2024-11-11T20:53:10,496 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:53:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T20:53:10,497 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358390497"}]},"ts":"1731358390497"} 2024-11-11T20:53:10,498 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:53:10,498 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 09438051ac0c3c1f634095c776d91044/C is initiating minor compaction (all files) 2024-11-11T20:53:10,498 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 09438051ac0c3c1f634095c776d91044/C in TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:10,498 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/697336280ee4441dbe4648b1d978ffc9, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/e5641ad004b14b91b248d9ad6dedc492, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/6666a37e15c3450f96b81fb3d58bc7c5, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/8ef334f8f7a04dabb0cc51f2e4498b78] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp, totalSize=48.9 K 2024-11-11T20:53:10,499 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 697336280ee4441dbe4648b1d978ffc9, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=648, earliestPutTs=1731358385627 2024-11-11T20:53:10,499 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5641ad004b14b91b248d9ad6dedc492, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=669, earliestPutTs=1731358385923 2024-11-11T20:53:10,499 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T20:53:10,499 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T20:53:10,499 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6666a37e15c3450f96b81fb3d58bc7c5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=683, earliestPutTs=1731358386842 2024-11-11T20:53:10,500 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ef334f8f7a04dabb0cc51f2e4498b78, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=714, earliestPutTs=1731358387573 2024-11-11T20:53:10,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T20:53:10,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=09438051ac0c3c1f634095c776d91044, UNASSIGN}] 2024-11-11T20:53:10,508 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=09438051ac0c3c1f634095c776d91044, UNASSIGN 2024-11-11T20:53:10,509 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=09438051ac0c3c1f634095c776d91044, regionState=CLOSING, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:10,510 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 09438051ac0c3c1f634095c776d91044#C#compaction#92 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:10,511 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/169d8de913e04857989ba1aa0ba3d4f0 is 50, key is test_row_0/C:col10/1731358387573/Put/seqid=0 2024-11-11T20:53:10,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=09438051ac0c3c1f634095c776d91044, UNASSIGN because future has completed 2024-11-11T20:53:10,512 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T20:53:10,513 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; CloseRegionProcedure 09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066}] 2024-11-11T20:53:10,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741929_1105 (size=13323) 2024-11-11T20:53:10,577 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T20:53:10,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T20:53:10,672 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(122): Close 09438051ac0c3c1f634095c776d91044 2024-11-11T20:53:10,672 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T20:53:10,674 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1722): Closing 09438051ac0c3c1f634095c776d91044, disabling compactions & flushes 2024-11-11T20:53:10,674 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1993): waiting for 1 compactions to complete for region TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:10,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T20:53:10,924 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/169d8de913e04857989ba1aa0ba3d4f0 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/169d8de913e04857989ba1aa0ba3d4f0 2024-11-11T20:53:10,930 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 09438051ac0c3c1f634095c776d91044/C of 09438051ac0c3c1f634095c776d91044 into 169d8de913e04857989ba1aa0ba3d4f0(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:10,930 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 09438051ac0c3c1f634095c776d91044: 2024-11-11T20:53:10,930 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:10,930 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., storeName=09438051ac0c3c1f634095c776d91044/C, priority=12, startTime=1731358390063; duration=0sec 2024-11-11T20:53:10,930 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:10,930 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. after waiting 0 ms 2024-11-11T20:53:10,930 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:10,930 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 09438051ac0c3c1f634095c776d91044:C 2024-11-11T20:53:10,930 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:10,930 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(2902): Flushing 09438051ac0c3c1f634095c776d91044 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-11T20:53:10,930 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=A 2024-11-11T20:53:10,930 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:10,931 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=B 2024-11-11T20:53:10,931 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:10,931 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 09438051ac0c3c1f634095c776d91044, store=C 2024-11-11T20:53:10,931 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:10,934 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/f21cd94756104389829981f533271451 is 50, key is test_row_0/A:col10/1731358390116/Put/seqid=0 2024-11-11T20:53:10,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741930_1106 (size=9857) 2024-11-11T20:53:11,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T20:53:11,340 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=724 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/f21cd94756104389829981f533271451 2024-11-11T20:53:11,355 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/6286414902784bb4b354ad83bbd55ac8 is 50, key is test_row_0/B:col10/1731358390116/Put/seqid=0 2024-11-11T20:53:11,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741931_1107 (size=9857) 2024-11-11T20:53:11,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T20:53:11,761 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=724 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/6286414902784bb4b354ad83bbd55ac8 2024-11-11T20:53:11,769 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/21933a509a664db9b0b95b64b00bda80 is 50, key is test_row_0/C:col10/1731358390116/Put/seqid=0 2024-11-11T20:53:11,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741932_1108 (size=9857) 2024-11-11T20:53:12,174 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=724 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/21933a509a664db9b0b95b64b00bda80 2024-11-11T20:53:12,180 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/A/f21cd94756104389829981f533271451 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/f21cd94756104389829981f533271451 2024-11-11T20:53:12,184 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/f21cd94756104389829981f533271451, entries=100, sequenceid=724, filesize=9.6 K 2024-11-11T20:53:12,185 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/B/6286414902784bb4b354ad83bbd55ac8 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/6286414902784bb4b354ad83bbd55ac8 2024-11-11T20:53:12,189 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/6286414902784bb4b354ad83bbd55ac8, entries=100, sequenceid=724, filesize=9.6 K 2024-11-11T20:53:12,190 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/.tmp/C/21933a509a664db9b0b95b64b00bda80 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/21933a509a664db9b0b95b64b00bda80 2024-11-11T20:53:12,194 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/21933a509a664db9b0b95b64b00bda80, entries=100, sequenceid=724, filesize=9.6 K 2024-11-11T20:53:12,195 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 09438051ac0c3c1f634095c776d91044 in 1265ms, sequenceid=724, compaction requested=false 2024-11-11T20:53:12,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/df5d55d214954f5d94aaa6c231a7c319, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/109d35677c5a4aaea40cbefc41087018, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/faea3aa34cec4215b371a4aaf7c30adf, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/456ef4e2fcf546a9a3f00274027d7e08, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/022cc05cae03401282f86f4a44ffdb51, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/b70bcce1ca714428bddbec7be843b08a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/d0df97357d614170a5acc4444fd919e6, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/dac4e0169f8e40ffa09d5d3f6d345242, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/dde2029dbb55466e90d9071fcfa9c66a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/d5e983c9f6c9473cab1245449beb0103, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/631cc022a7824fd7b8ae36f54dd91f09, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/f5323228b7794195b90fd74f735ab3a8, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/569cbba743b5463d82abf94da4d06d69, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/8a0719cf2ad74deebfbf0acbd9036af0, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/2782b7764f4c453888dacdc60c3564cb, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/9098f4bb663e4b2385dba3146d2c99f2, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/09f55155819f42389b23feb2350f39b9, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/fe9d9bfeca6f4313aa405f95adb38103, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/4538847c1c4a4a5e96725747ca79cd8b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/f2c88d22774f434f9712e0e25573ce83, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/bc8d904f71eb43f2a1556ddd9a74619d, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/68b598035a24447b8abfdd727156ac1a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/df02e313a1d74cd9a3efc91a0fcf0a19, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/8d135ae066f74003b083ddacf36d881b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/185c7fa2bcdb48c7aa72a6dd2505f44a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/c3f035f73cfe462fadc87e0b7ddf91c2, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/6f9c3780567f40b08da4c3aebaf33813, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/14eae057530f4703ae38162a40d6a308, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/002381ece1d24f33bc246529d646bbde, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/caeeff18b8fe42a1998a21108024bfb0] to archive 2024-11-11T20:53:12,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:53:12,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/df5d55d214954f5d94aaa6c231a7c319 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/df5d55d214954f5d94aaa6c231a7c319 2024-11-11T20:53:12,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/109d35677c5a4aaea40cbefc41087018 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/109d35677c5a4aaea40cbefc41087018 2024-11-11T20:53:12,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/faea3aa34cec4215b371a4aaf7c30adf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/faea3aa34cec4215b371a4aaf7c30adf 2024-11-11T20:53:12,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/456ef4e2fcf546a9a3f00274027d7e08 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/456ef4e2fcf546a9a3f00274027d7e08 2024-11-11T20:53:12,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/022cc05cae03401282f86f4a44ffdb51 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/022cc05cae03401282f86f4a44ffdb51 2024-11-11T20:53:12,210 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/b70bcce1ca714428bddbec7be843b08a to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/b70bcce1ca714428bddbec7be843b08a 2024-11-11T20:53:12,211 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/d0df97357d614170a5acc4444fd919e6 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/d0df97357d614170a5acc4444fd919e6 2024-11-11T20:53:12,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/dac4e0169f8e40ffa09d5d3f6d345242 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/dac4e0169f8e40ffa09d5d3f6d345242 2024-11-11T20:53:12,213 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/dde2029dbb55466e90d9071fcfa9c66a to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/dde2029dbb55466e90d9071fcfa9c66a 2024-11-11T20:53:12,214 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/d5e983c9f6c9473cab1245449beb0103 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/d5e983c9f6c9473cab1245449beb0103 2024-11-11T20:53:12,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/631cc022a7824fd7b8ae36f54dd91f09 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/631cc022a7824fd7b8ae36f54dd91f09 2024-11-11T20:53:12,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/f5323228b7794195b90fd74f735ab3a8 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/f5323228b7794195b90fd74f735ab3a8 2024-11-11T20:53:12,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/569cbba743b5463d82abf94da4d06d69 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/569cbba743b5463d82abf94da4d06d69 2024-11-11T20:53:12,218 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/8a0719cf2ad74deebfbf0acbd9036af0 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/8a0719cf2ad74deebfbf0acbd9036af0 2024-11-11T20:53:12,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/2782b7764f4c453888dacdc60c3564cb to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/2782b7764f4c453888dacdc60c3564cb 2024-11-11T20:53:12,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/9098f4bb663e4b2385dba3146d2c99f2 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/9098f4bb663e4b2385dba3146d2c99f2 2024-11-11T20:53:12,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/09f55155819f42389b23feb2350f39b9 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/09f55155819f42389b23feb2350f39b9 2024-11-11T20:53:12,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/fe9d9bfeca6f4313aa405f95adb38103 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/fe9d9bfeca6f4313aa405f95adb38103 2024-11-11T20:53:12,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/4538847c1c4a4a5e96725747ca79cd8b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/4538847c1c4a4a5e96725747ca79cd8b 2024-11-11T20:53:12,226 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/f2c88d22774f434f9712e0e25573ce83 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/f2c88d22774f434f9712e0e25573ce83 2024-11-11T20:53:12,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/bc8d904f71eb43f2a1556ddd9a74619d to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/bc8d904f71eb43f2a1556ddd9a74619d 2024-11-11T20:53:12,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/68b598035a24447b8abfdd727156ac1a to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/68b598035a24447b8abfdd727156ac1a 2024-11-11T20:53:12,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/df02e313a1d74cd9a3efc91a0fcf0a19 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/df02e313a1d74cd9a3efc91a0fcf0a19 2024-11-11T20:53:12,232 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/8d135ae066f74003b083ddacf36d881b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/8d135ae066f74003b083ddacf36d881b 2024-11-11T20:53:12,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/185c7fa2bcdb48c7aa72a6dd2505f44a to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/185c7fa2bcdb48c7aa72a6dd2505f44a 2024-11-11T20:53:12,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/c3f035f73cfe462fadc87e0b7ddf91c2 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/c3f035f73cfe462fadc87e0b7ddf91c2 2024-11-11T20:53:12,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/6f9c3780567f40b08da4c3aebaf33813 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/6f9c3780567f40b08da4c3aebaf33813 2024-11-11T20:53:12,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/14eae057530f4703ae38162a40d6a308 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/14eae057530f4703ae38162a40d6a308 2024-11-11T20:53:12,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/002381ece1d24f33bc246529d646bbde to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/002381ece1d24f33bc246529d646bbde 2024-11-11T20:53:12,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/caeeff18b8fe42a1998a21108024bfb0 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/caeeff18b8fe42a1998a21108024bfb0 2024-11-11T20:53:12,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/8fad264debb04c268ba99f6e4f75b0fd, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/6f7ccff10acc4528a8df0aeba4ef881a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/ab14cf231dc34302bb2409951a840918, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/948b8b95adb54efe937a2e9cd2021c61, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/1f1ffbf7e2474bfa990bb6255f6a4306, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/8f1ff165af6848e5af216fb1254d4cec, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/2ef10c144cfb4363841404cc71645f6a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/983c4eea0c92447a9ea37b34a1823314, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/b11c1cf0e79446d79936022edd3920ec, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/3fbb6265cacb469386225212a4ffa8e8, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/290f72a9031b488c84438e0e197470a2, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/f7bf92c447f2439cbdfe4f17aac33a36, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/2ec52e5dd11d43e9950a7e7406e5eb97, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/b66b08156c494898b6c9521d568ab4f0, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/c16104b7816548f1a2681fa2a7b3d9e5, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/b2fa476601fb492297a69f69d46b9147, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/a5000c108f1449cc883ff3fdc0604a21, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/366ef1618e764b67b8ea876944cc8c33, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/176c60c329514f86b84ae5094e96cb12, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/0bc19ea98d9841efba5587a097e75304, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/c950497ebd1845b49f68062199808701, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/96985a4ce41548aabb15dc580f3f3fc0, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/90afc8255fa14fe5a8d5f873ebef0eda, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/79dde07d2ea94a91928c89bea285f2e3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/151fc54ef83b4083b9df93d921b7de8c, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/45d7d8cdcfc74d18ba71464f2db2c621, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/87c7062f7f6c4bb2bda9624071da8386, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/9bd4bd4faca84bf892d6f883bc2f9112, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/e43a7574361b4fe5b3425290ce3255b3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/8df5bb46bdff4962b61827b0089683e0] to archive 2024-11-11T20:53:12,262 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:53:12,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/8fad264debb04c268ba99f6e4f75b0fd to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/8fad264debb04c268ba99f6e4f75b0fd 2024-11-11T20:53:12,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/6f7ccff10acc4528a8df0aeba4ef881a to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/6f7ccff10acc4528a8df0aeba4ef881a 2024-11-11T20:53:12,267 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/ab14cf231dc34302bb2409951a840918 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/ab14cf231dc34302bb2409951a840918 2024-11-11T20:53:12,268 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/948b8b95adb54efe937a2e9cd2021c61 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/948b8b95adb54efe937a2e9cd2021c61 2024-11-11T20:53:12,270 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/1f1ffbf7e2474bfa990bb6255f6a4306 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/1f1ffbf7e2474bfa990bb6255f6a4306 2024-11-11T20:53:12,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/8f1ff165af6848e5af216fb1254d4cec to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/8f1ff165af6848e5af216fb1254d4cec 2024-11-11T20:53:12,273 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/2ef10c144cfb4363841404cc71645f6a to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/2ef10c144cfb4363841404cc71645f6a 2024-11-11T20:53:12,275 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/983c4eea0c92447a9ea37b34a1823314 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/983c4eea0c92447a9ea37b34a1823314 2024-11-11T20:53:12,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/b11c1cf0e79446d79936022edd3920ec to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/b11c1cf0e79446d79936022edd3920ec 2024-11-11T20:53:12,277 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/3fbb6265cacb469386225212a4ffa8e8 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/3fbb6265cacb469386225212a4ffa8e8 2024-11-11T20:53:12,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/290f72a9031b488c84438e0e197470a2 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/290f72a9031b488c84438e0e197470a2 2024-11-11T20:53:12,280 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/f7bf92c447f2439cbdfe4f17aac33a36 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/f7bf92c447f2439cbdfe4f17aac33a36 2024-11-11T20:53:12,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/2ec52e5dd11d43e9950a7e7406e5eb97 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/2ec52e5dd11d43e9950a7e7406e5eb97 2024-11-11T20:53:12,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/b66b08156c494898b6c9521d568ab4f0 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/b66b08156c494898b6c9521d568ab4f0 2024-11-11T20:53:12,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/c16104b7816548f1a2681fa2a7b3d9e5 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/c16104b7816548f1a2681fa2a7b3d9e5 2024-11-11T20:53:12,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/b2fa476601fb492297a69f69d46b9147 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/b2fa476601fb492297a69f69d46b9147 2024-11-11T20:53:12,289 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/a5000c108f1449cc883ff3fdc0604a21 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/a5000c108f1449cc883ff3fdc0604a21 2024-11-11T20:53:12,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/366ef1618e764b67b8ea876944cc8c33 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/366ef1618e764b67b8ea876944cc8c33 2024-11-11T20:53:12,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/176c60c329514f86b84ae5094e96cb12 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/176c60c329514f86b84ae5094e96cb12 2024-11-11T20:53:12,293 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/0bc19ea98d9841efba5587a097e75304 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/0bc19ea98d9841efba5587a097e75304 2024-11-11T20:53:12,295 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/c950497ebd1845b49f68062199808701 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/c950497ebd1845b49f68062199808701 2024-11-11T20:53:12,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/96985a4ce41548aabb15dc580f3f3fc0 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/96985a4ce41548aabb15dc580f3f3fc0 2024-11-11T20:53:12,297 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/90afc8255fa14fe5a8d5f873ebef0eda to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/90afc8255fa14fe5a8d5f873ebef0eda 2024-11-11T20:53:12,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/79dde07d2ea94a91928c89bea285f2e3 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/79dde07d2ea94a91928c89bea285f2e3 2024-11-11T20:53:12,299 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/151fc54ef83b4083b9df93d921b7de8c to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/151fc54ef83b4083b9df93d921b7de8c 2024-11-11T20:53:12,300 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/45d7d8cdcfc74d18ba71464f2db2c621 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/45d7d8cdcfc74d18ba71464f2db2c621 2024-11-11T20:53:12,302 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/87c7062f7f6c4bb2bda9624071da8386 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/87c7062f7f6c4bb2bda9624071da8386 2024-11-11T20:53:12,303 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/9bd4bd4faca84bf892d6f883bc2f9112 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/9bd4bd4faca84bf892d6f883bc2f9112 2024-11-11T20:53:12,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/e43a7574361b4fe5b3425290ce3255b3 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/e43a7574361b4fe5b3425290ce3255b3 2024-11-11T20:53:12,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/8df5bb46bdff4962b61827b0089683e0 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/8df5bb46bdff4962b61827b0089683e0 2024-11-11T20:53:12,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/6fd5a18b8cba4c25908d8bde486d2f04, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/571d458708e746a18ce50136078577b0, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/dcf88198ea23431a81d464a996e12116, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/c05b4314b8e14591b36d31a703402991, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/4256db5113814bb780dfaf267f56cce2, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/86b3501685f741e0929419b509a50024, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/3b0d9d08303f459e974f97357ddc2fec, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/a5bbfc8e2e114a00a6a31bd98d0678db, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/5463bfcd5f8b44d3bb259a878c57a163, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/e0fa4a2a8849471888178cd41252c9f8, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/14e67390edcb427a98343dd80f3b6c30, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/fccef71ef0b14266af28347d55a76180, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/37e60aa8fa734564a19a0834d3c0f71f, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/5cb7ecf97ca34f68ae3c97f83d678821, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/816fd990fc2b4172be55844eb517fe94, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/666d248b632d46d7ab511f601cd14a83, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/46d5c775659a46d0bd9ae1d3cd2e6072, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/43f25c4495b8422d82529ee1379d690b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/d5d81f4a269b4ad480df2d031f7d73f6, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/2eb88bb4726b4d4ea59bafdd2f56ccca, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/b6ac927f9cf744498fd1e9c800caa6cb, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/58efceee1c8c40839121d9dbb7894396, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/c9c9d582bc7b4ce9bde0f85c24bf9079, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/53cee8e1caeb42cd9281b6eb2b06bb16, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/e8843c44fea445f9816b44636d741cbc, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/697336280ee4441dbe4648b1d978ffc9, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/9489053f5d544dd0a1d3b18b2b8d977f, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/e5641ad004b14b91b248d9ad6dedc492, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/6666a37e15c3450f96b81fb3d58bc7c5, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/8ef334f8f7a04dabb0cc51f2e4498b78] to archive 2024-11-11T20:53:12,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:53:12,313 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/6fd5a18b8cba4c25908d8bde486d2f04 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/6fd5a18b8cba4c25908d8bde486d2f04 2024-11-11T20:53:12,314 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/571d458708e746a18ce50136078577b0 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/571d458708e746a18ce50136078577b0 2024-11-11T20:53:12,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/dcf88198ea23431a81d464a996e12116 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/dcf88198ea23431a81d464a996e12116 2024-11-11T20:53:12,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/c05b4314b8e14591b36d31a703402991 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/c05b4314b8e14591b36d31a703402991 2024-11-11T20:53:12,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/4256db5113814bb780dfaf267f56cce2 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/4256db5113814bb780dfaf267f56cce2 2024-11-11T20:53:12,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/86b3501685f741e0929419b509a50024 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/86b3501685f741e0929419b509a50024 2024-11-11T20:53:12,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/3b0d9d08303f459e974f97357ddc2fec to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/3b0d9d08303f459e974f97357ddc2fec 2024-11-11T20:53:12,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/a5bbfc8e2e114a00a6a31bd98d0678db to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/a5bbfc8e2e114a00a6a31bd98d0678db 2024-11-11T20:53:12,324 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/5463bfcd5f8b44d3bb259a878c57a163 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/5463bfcd5f8b44d3bb259a878c57a163 2024-11-11T20:53:12,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/e0fa4a2a8849471888178cd41252c9f8 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/e0fa4a2a8849471888178cd41252c9f8 2024-11-11T20:53:12,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/14e67390edcb427a98343dd80f3b6c30 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/14e67390edcb427a98343dd80f3b6c30 2024-11-11T20:53:12,331 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/fccef71ef0b14266af28347d55a76180 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/fccef71ef0b14266af28347d55a76180 2024-11-11T20:53:12,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/37e60aa8fa734564a19a0834d3c0f71f to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/37e60aa8fa734564a19a0834d3c0f71f 2024-11-11T20:53:12,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/5cb7ecf97ca34f68ae3c97f83d678821 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/5cb7ecf97ca34f68ae3c97f83d678821 2024-11-11T20:53:12,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/816fd990fc2b4172be55844eb517fe94 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/816fd990fc2b4172be55844eb517fe94 2024-11-11T20:53:12,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/666d248b632d46d7ab511f601cd14a83 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/666d248b632d46d7ab511f601cd14a83 2024-11-11T20:53:12,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/46d5c775659a46d0bd9ae1d3cd2e6072 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/46d5c775659a46d0bd9ae1d3cd2e6072 2024-11-11T20:53:12,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/43f25c4495b8422d82529ee1379d690b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/43f25c4495b8422d82529ee1379d690b 2024-11-11T20:53:12,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/d5d81f4a269b4ad480df2d031f7d73f6 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/d5d81f4a269b4ad480df2d031f7d73f6 2024-11-11T20:53:12,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/2eb88bb4726b4d4ea59bafdd2f56ccca to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/2eb88bb4726b4d4ea59bafdd2f56ccca 2024-11-11T20:53:12,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/b6ac927f9cf744498fd1e9c800caa6cb to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/b6ac927f9cf744498fd1e9c800caa6cb 2024-11-11T20:53:12,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/58efceee1c8c40839121d9dbb7894396 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/58efceee1c8c40839121d9dbb7894396 2024-11-11T20:53:12,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/c9c9d582bc7b4ce9bde0f85c24bf9079 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/c9c9d582bc7b4ce9bde0f85c24bf9079 2024-11-11T20:53:12,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/53cee8e1caeb42cd9281b6eb2b06bb16 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/53cee8e1caeb42cd9281b6eb2b06bb16 2024-11-11T20:53:12,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/e8843c44fea445f9816b44636d741cbc to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/e8843c44fea445f9816b44636d741cbc 2024-11-11T20:53:12,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/697336280ee4441dbe4648b1d978ffc9 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/697336280ee4441dbe4648b1d978ffc9 2024-11-11T20:53:12,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/9489053f5d544dd0a1d3b18b2b8d977f to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/9489053f5d544dd0a1d3b18b2b8d977f 2024-11-11T20:53:12,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/e5641ad004b14b91b248d9ad6dedc492 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/e5641ad004b14b91b248d9ad6dedc492 2024-11-11T20:53:12,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/6666a37e15c3450f96b81fb3d58bc7c5 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/6666a37e15c3450f96b81fb3d58bc7c5 2024-11-11T20:53:12,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/8ef334f8f7a04dabb0cc51f2e4498b78 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/8ef334f8f7a04dabb0cc51f2e4498b78 2024-11-11T20:53:12,365 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/recovered.edits/727.seqid, newMaxSeqId=727, maxSeqId=1 2024-11-11T20:53:12,367 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. 2024-11-11T20:53:12,367 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] regionserver.HRegion(1676): Region close journal for 09438051ac0c3c1f634095c776d91044: Waiting for close lock at 1731358390674Running coprocessor pre-close hooks at 1731358390674Disabling compacts and flushes for region at 1731358390674Disabling writes for close at 1731358390930 (+256 ms)Obtaining lock to block concurrent updates at 1731358390930Preparing flush snapshotting stores in 09438051ac0c3c1f634095c776d91044 at 1731358390930Finished memstore snapshotting TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731358390931 (+1 ms)Flushing stores of TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044. at 1731358390931Flushing 09438051ac0c3c1f634095c776d91044/A: creating writer at 1731358390931Flushing 09438051ac0c3c1f634095c776d91044/A: appending metadata at 1731358390934 (+3 ms)Flushing 09438051ac0c3c1f634095c776d91044/A: closing flushed file at 1731358390934Flushing 09438051ac0c3c1f634095c776d91044/B: creating writer at 1731358391351 (+417 ms)Flushing 09438051ac0c3c1f634095c776d91044/B: appending metadata at 1731358391355 (+4 ms)Flushing 09438051ac0c3c1f634095c776d91044/B: closing flushed file at 1731358391355Flushing 09438051ac0c3c1f634095c776d91044/C: creating writer at 1731358391766 (+411 ms)Flushing 09438051ac0c3c1f634095c776d91044/C: appending metadata at 1731358391768 (+2 ms)Flushing 09438051ac0c3c1f634095c776d91044/C: closing flushed file at 1731358391769 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2769d226: reopening flushed file at 1731358392179 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74b0a6e8: reopening flushed file at 1731358392184 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bb3ad90: reopening flushed file at 1731358392190 (+6 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 09438051ac0c3c1f634095c776d91044 in 1265ms, sequenceid=724, compaction requested=false at 1731358392195 (+5 ms)Writing region close event to WAL at 1731358392361 (+166 ms)Running coprocessor post-close hooks at 1731358392366 (+5 ms)Closed at 1731358392367 (+1 ms) 2024-11-11T20:53:12,370 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=22}] handler.UnassignRegionHandler(157): Closed 09438051ac0c3c1f634095c776d91044 2024-11-11T20:53:12,370 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=21 updating hbase:meta row=09438051ac0c3c1f634095c776d91044, regionState=CLOSED 2024-11-11T20:53:12,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=22, ppid=21, state=RUNNABLE, hasLock=false; CloseRegionProcedure 09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 because future has completed 2024-11-11T20:53:12,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-11T20:53:12,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; CloseRegionProcedure 09438051ac0c3c1f634095c776d91044, server=51ca66f7ee3c,44669,1731358362066 in 1.8600 sec 2024-11-11T20:53:12,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-11T20:53:12,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=09438051ac0c3c1f634095c776d91044, UNASSIGN in 1.8680 sec 2024-11-11T20:53:12,379 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-11T20:53:12,379 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8770 sec 2024-11-11T20:53:12,381 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358392381"}]},"ts":"1731358392381"} 2024-11-11T20:53:12,383 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T20:53:12,383 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T20:53:12,385 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8920 sec 2024-11-11T20:53:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T20:53:12,646 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:53:12,650 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T20:53:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:53:12,657 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=23, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:53:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T20:53:12,659 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=23, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:53:12,664 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044 2024-11-11T20:53:12,669 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A, FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B, FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C, FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/recovered.edits] 2024-11-11T20:53:12,673 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/5dc8a4d752f440d6a1aeafe47d1a8702 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/5dc8a4d752f440d6a1aeafe47d1a8702 2024-11-11T20:53:12,675 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/f21cd94756104389829981f533271451 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/A/f21cd94756104389829981f533271451 2024-11-11T20:53:12,679 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/6286414902784bb4b354ad83bbd55ac8 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/6286414902784bb4b354ad83bbd55ac8 2024-11-11T20:53:12,681 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/ab665da0c8b0440b84281f0062fcc09b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/B/ab665da0c8b0440b84281f0062fcc09b 2024-11-11T20:53:12,685 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/169d8de913e04857989ba1aa0ba3d4f0 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/169d8de913e04857989ba1aa0ba3d4f0 2024-11-11T20:53:12,687 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/21933a509a664db9b0b95b64b00bda80 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/C/21933a509a664db9b0b95b64b00bda80 2024-11-11T20:53:12,691 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/recovered.edits/727.seqid to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044/recovered.edits/727.seqid 2024-11-11T20:53:12,692 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/09438051ac0c3c1f634095c776d91044 2024-11-11T20:53:12,692 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T20:53:12,698 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=23, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:53:12,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-11T20:53:12,708 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T20:53:12,745 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T20:53:12,747 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=23, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:53:12,747 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T20:53:12,747 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731358392747"}]},"ts":"9223372036854775807"} 2024-11-11T20:53:12,752 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T20:53:12,752 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 09438051ac0c3c1f634095c776d91044, NAME => 'TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T20:53:12,752 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T20:53:12,752 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731358392752"}]},"ts":"9223372036854775807"} 2024-11-11T20:53:12,755 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T20:53:12,756 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=23, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:53:12,758 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 105 msec 2024-11-11T20:53:12,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T20:53:12,767 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T20:53:12,767 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:53:12,777 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=250 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;51ca66f7ee3c:44669-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2048145703_22 at /127.0.0.1:58670 [Waiting for operation #63] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2048145703_22 at /127.0.0.1:34690 [Waiting for operation #254] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-634654397_22 at /127.0.0.1:49094 [Waiting for operation #269] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-634654397_22 at /127.0.0.1:38942 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=461 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=378 (was 256) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4432 (was 3315) - AvailableMemoryMB LEAK? - 2024-11-11T20:53:12,786 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=250, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=378, ProcessCount=11, AvailableMemoryMB=4431 2024-11-11T20:53:12,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T20:53:12,789 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T20:53:12,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T20:53:12,791 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T20:53:12,791 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:12,792 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 24 2024-11-11T20:53:12,792 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T20:53:12,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-11T20:53:12,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741933_1109 (size=1061) 2024-11-11T20:53:12,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-11T20:53:13,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-11T20:53:13,201 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23 2024-11-11T20:53:13,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741934_1110 (size=53) 2024-11-11T20:53:13,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-11T20:53:13,608 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:53:13,608 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 759cf260483d400f5615c66d4fd85216, disabling compactions & flushes 2024-11-11T20:53:13,608 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:13,608 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:13,608 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. after waiting 0 ms 2024-11-11T20:53:13,608 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:13,609 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:13,609 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 759cf260483d400f5615c66d4fd85216: Waiting for close lock at 1731358393608Disabling compacts and flushes for region at 1731358393608Disabling writes for close at 1731358393608Writing region close event to WAL at 1731358393608Closed at 1731358393608 2024-11-11T20:53:13,611 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T20:53:13,611 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731358393611"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358393611"}]},"ts":"1731358393611"} 2024-11-11T20:53:13,615 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T20:53:13,617 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T20:53:13,618 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358393618"}]},"ts":"1731358393618"} 2024-11-11T20:53:13,621 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T20:53:13,621 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=759cf260483d400f5615c66d4fd85216, ASSIGN}] 2024-11-11T20:53:13,622 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=759cf260483d400f5615c66d4fd85216, ASSIGN 2024-11-11T20:53:13,623 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=759cf260483d400f5615c66d4fd85216, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,44669,1731358362066; forceNewPlan=false, retain=false 2024-11-11T20:53:13,774 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=759cf260483d400f5615c66d4fd85216, regionState=OPENING, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:13,778 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=25, ppid=24, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=759cf260483d400f5615c66d4fd85216, ASSIGN because future has completed 2024-11-11T20:53:13,779 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066}] 2024-11-11T20:53:13,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-11T20:53:13,937 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:13,937 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7752): Opening region: {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:53:13,937 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:13,937 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:53:13,938 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7794): checking encryption for 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:13,938 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(7797): checking classloading for 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:13,939 INFO [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:13,940 INFO [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T20:53:13,941 INFO [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 759cf260483d400f5615c66d4fd85216 columnFamilyName A 2024-11-11T20:53:13,941 DEBUG [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:13,942 INFO [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] regionserver.HStore(327): Store=759cf260483d400f5615c66d4fd85216/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:53:13,942 INFO [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:13,943 INFO [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T20:53:13,943 INFO [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 759cf260483d400f5615c66d4fd85216 columnFamilyName B 2024-11-11T20:53:13,943 DEBUG [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:13,944 INFO [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] regionserver.HStore(327): Store=759cf260483d400f5615c66d4fd85216/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:53:13,944 INFO [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:13,945 INFO [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T20:53:13,945 INFO [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 759cf260483d400f5615c66d4fd85216 columnFamilyName C 2024-11-11T20:53:13,945 DEBUG [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:13,945 INFO [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] regionserver.HStore(327): Store=759cf260483d400f5615c66d4fd85216/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:53:13,945 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:13,946 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1038): replaying wal for 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:13,946 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:13,947 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:13,948 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1048): stopping wal replay for 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:13,948 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1060): Cleaning up temporary data for 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:13,948 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T20:53:13,950 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1093): writing seq id for 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:13,952 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:53:13,953 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1114): Opened 759cf260483d400f5615c66d4fd85216; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73700390, jitterRate=0.09822139143943787}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T20:53:13,953 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:13,954 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegion(1006): Region open journal for 759cf260483d400f5615c66d4fd85216: Running coprocessor pre-open hook at 1731358393938Writing region info on filesystem at 1731358393938Initializing all the Stores at 1731358393939 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358393939Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358393939Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358393939Cleaning up temporary data from old regions at 1731358393948 (+9 ms)Running coprocessor post-open hooks at 1731358393953 (+5 ms)Region opened successfully at 1731358393954 (+1 ms) 2024-11-11T20:53:13,955 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., pid=26, masterSystemTime=1731358393932 2024-11-11T20:53:13,957 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:13,957 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=26}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:13,958 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=25 updating hbase:meta row=759cf260483d400f5615c66d4fd85216, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:13,959 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=26, ppid=25, state=RUNNABLE, hasLock=false; OpenRegionProcedure 759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 because future has completed 2024-11-11T20:53:13,962 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-11T20:53:13,962 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; OpenRegionProcedure 759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 in 182 msec 2024-11-11T20:53:13,965 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=25, resume processing ppid=24 2024-11-11T20:53:13,965 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, ppid=24, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=759cf260483d400f5615c66d4fd85216, ASSIGN in 341 msec 2024-11-11T20:53:13,965 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T20:53:13,966 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358393965"}]},"ts":"1731358393965"} 2024-11-11T20:53:13,967 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T20:53:13,968 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=24, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T20:53:13,970 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1790 sec 2024-11-11T20:53:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-11-11T20:53:14,947 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:53:14,948 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@736030eb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:14,948 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:53:14,948 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:53:14,950 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:53:14,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:53:14,950 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:53:14,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55f1c931, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:14,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:53:14,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:53:14,951 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:14,953 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51890, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:53:14,954 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13c68170, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:14,954 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:53:14,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:53:14,957 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:53:14,958 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49476, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:53:14,960 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T20:53:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 51ca66f7ee3c,38429,1731358361462 2024-11-11T20:53:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3accb36a 2024-11-11T20:53:14,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T20:53:14,963 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51902, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T20:53:14,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T20:53:14,970 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T20:53:14,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T20:53:14,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741935_1111 (size=1097) 2024-11-11T20:53:15,393 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-11T20:53:15,393 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-11T20:53:15,399 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T20:53:15,406 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=759cf260483d400f5615c66d4fd85216, REOPEN/MOVE}] 2024-11-11T20:53:15,407 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=759cf260483d400f5615c66d4fd85216, REOPEN/MOVE 2024-11-11T20:53:15,408 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=759cf260483d400f5615c66d4fd85216, regionState=CLOSING, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:15,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=759cf260483d400f5615c66d4fd85216, REOPEN/MOVE because future has completed 2024-11-11T20:53:15,410 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T20:53:15,410 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066}] 2024-11-11T20:53:15,563 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:15,563 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T20:53:15,564 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 759cf260483d400f5615c66d4fd85216, disabling compactions & flushes 2024-11-11T20:53:15,564 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:15,564 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:15,564 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. after waiting 0 ms 2024-11-11T20:53:15,564 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:15,573 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:53:15,574 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:15,574 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 759cf260483d400f5615c66d4fd85216: Waiting for close lock at 1731358395564Running coprocessor pre-close hooks at 1731358395564Disabling compacts and flushes for region at 1731358395564Disabling writes for close at 1731358395564Writing region close event to WAL at 1731358395565 (+1 ms)Running coprocessor post-close hooks at 1731358395574 (+9 ms)Closed at 1731358395574 2024-11-11T20:53:15,575 WARN [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionServer(3299): Not adding moved region record: 759cf260483d400f5615c66d4fd85216 to self. 2024-11-11T20:53:15,577 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:15,578 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=759cf260483d400f5615c66d4fd85216, regionState=CLOSED 2024-11-11T20:53:15,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 because future has completed 2024-11-11T20:53:15,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-11T20:53:15,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 in 171 msec 2024-11-11T20:53:15,583 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=759cf260483d400f5615c66d4fd85216, REOPEN/MOVE; state=CLOSED, location=51ca66f7ee3c,44669,1731358362066; forceNewPlan=false, retain=true 2024-11-11T20:53:15,734 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=759cf260483d400f5615c66d4fd85216, regionState=OPENING, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:15,737 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=759cf260483d400f5615c66d4fd85216, REOPEN/MOVE because future has completed 2024-11-11T20:53:15,738 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066}] 2024-11-11T20:53:15,895 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:15,895 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7752): Opening region: {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:53:15,896 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:15,896 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:53:15,896 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7794): checking encryption for 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:15,896 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(7797): checking classloading for 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:15,901 INFO [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:15,902 INFO [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T20:53:15,909 INFO [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 759cf260483d400f5615c66d4fd85216 columnFamilyName A 2024-11-11T20:53:15,912 DEBUG [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:15,913 INFO [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] regionserver.HStore(327): Store=759cf260483d400f5615c66d4fd85216/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:53:15,913 INFO [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:15,914 INFO [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T20:53:15,914 INFO [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 759cf260483d400f5615c66d4fd85216 columnFamilyName B 2024-11-11T20:53:15,914 DEBUG [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:15,915 INFO [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] regionserver.HStore(327): Store=759cf260483d400f5615c66d4fd85216/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:53:15,915 INFO [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:15,915 INFO [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T20:53:15,916 INFO [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 759cf260483d400f5615c66d4fd85216 columnFamilyName C 2024-11-11T20:53:15,916 DEBUG [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:15,916 INFO [StoreOpener-759cf260483d400f5615c66d4fd85216-1 {}] regionserver.HStore(327): Store=759cf260483d400f5615c66d4fd85216/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:53:15,916 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:15,916 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1038): replaying wal for 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:15,917 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:15,918 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:15,919 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1048): stopping wal replay for 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:15,919 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1060): Cleaning up temporary data for 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:15,920 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T20:53:15,921 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1093): writing seq id for 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:15,922 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1114): Opened 759cf260483d400f5615c66d4fd85216; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67956150, jitterRate=0.012625545263290405}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T20:53:15,922 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:15,924 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegion(1006): Region open journal for 759cf260483d400f5615c66d4fd85216: Running coprocessor pre-open hook at 1731358395897Writing region info on filesystem at 1731358395897Initializing all the Stores at 1731358395898 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358395899 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358395899Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358395899Cleaning up temporary data from old regions at 1731358395919 (+20 ms)Running coprocessor post-open hooks at 1731358395922 (+3 ms)Region opened successfully at 1731358395923 (+1 ms) 2024-11-11T20:53:15,924 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., pid=31, masterSystemTime=1731358395891 2024-11-11T20:53:15,926 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:15,926 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=31}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:15,927 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=759cf260483d400f5615c66d4fd85216, regionState=OPEN, openSeqNum=5, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:15,929 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=29, state=RUNNABLE, hasLock=false; OpenRegionProcedure 759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 because future has completed 2024-11-11T20:53:15,931 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=29 2024-11-11T20:53:15,931 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=29, state=SUCCESS, hasLock=false; OpenRegionProcedure 759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 in 193 msec 2024-11-11T20:53:15,933 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-11T20:53:15,933 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=759cf260483d400f5615c66d4fd85216, REOPEN/MOVE in 525 msec 2024-11-11T20:53:15,936 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-11T20:53:15,936 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 536 msec 2024-11-11T20:53:15,939 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 965 msec 2024-11-11T20:53:15,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T20:53:15,941 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:53:15,941 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65b7b5d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:15,941 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:53:15,941 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:53:15,942 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:53:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:53:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:53:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a28c26a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:53:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:53:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:15,944 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:53:15,945 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@289e5f6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:15,946 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a1845b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:15,946 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:53:15,946 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:53:15,947 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:53:15,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:53:15,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:53:15,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@441bf5fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:15,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:53:15,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:53:15,947 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:15,948 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51930, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:53:15,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67618fa3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:15,950 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b0484de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:15,950 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:53:15,950 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:53:15,951 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:53:15,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:53:15,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:53:15,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@290551f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:15,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:53:15,951 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:53:15,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:15,952 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:53:15,953 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@454b9828, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:15,954 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71151d6d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:15,954 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:53:15,954 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:53:15,955 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:53:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:53:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:53:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1980727, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:15,955 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:53:15,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:53:15,956 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:15,956 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:53:15,957 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1158de5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:15,957 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@677aa7ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:15,958 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:53:15,958 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:53:15,958 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:53:15,958 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:53:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:53:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@473c8ff0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:53:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:53:15,959 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:15,960 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52010, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:53:15,960 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c9466ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:15,961 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dd18334, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:15,961 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:53:15,961 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:53:15,962 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:53:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:53:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:53:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@733bdc81, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:53:15,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:53:15,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:15,963 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:53:15,964 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30fe0df6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:15,964 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52c57d01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:15,964 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:53:15,964 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:53:15,965 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:53:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:53:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:53:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bccbc99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:15,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:53:15,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:53:15,966 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:15,967 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52040, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:53:15,967 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@341602f3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:15,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b465b92, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:15,968 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:53:15,968 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:53:15,968 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:53:15,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:53:15,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:53:15,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4aa9c030, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:15,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:53:15,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:53:15,969 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:15,970 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52060, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:53:15,970 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e57bfd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:15,971 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@711c9f1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:15,971 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:53:15,971 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:53:15,972 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:53:15,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:53:15,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:53:15,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2247b8b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:15,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:53:15,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:53:15,972 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:15,973 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:53:15,973 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@219244cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:15,974 DEBUG [Thread-578 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:53:15,974 DEBUG [Thread-580 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:53:15,974 DEBUG [Thread-576 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:53:15,975 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:53:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:53:15,975 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:53:15,976 DEBUG [Thread-591 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:53:15,976 DEBUG [Thread-587 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:53:15,976 DEBUG [Thread-589 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:53:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:53:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:53:15,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees 2024-11-11T20:53:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:53:15,976 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:53:15,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:53:15,977 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:53:15,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:53:15,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:53:15,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:53:15,977 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:53:15,978 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49478, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:53:15,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-11T20:53:15,978 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49480, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:53:15,978 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49508, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:53:15,979 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49512, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:53:15,979 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49506, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:53:15,979 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49528, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:53:15,979 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=32, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:53:15,980 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=32, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:53:15,980 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:53:15,982 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:53:15,983 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:53:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:53:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:53:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:53:15,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:53:15,985 DEBUG [Thread-582 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:53:15,985 DEBUG [Thread-584 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:53:15,985 DEBUG [Thread-593 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:53:15,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:53:15,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:53:15,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:53:15,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:53:15,986 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:53:15,986 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:53:15,987 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49536, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:53:15,988 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:53:15,989 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:53:15,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:53:15,990 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:53:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:53:15,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:16,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 759cf260483d400f5615c66d4fd85216 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T20:53:16,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=A 2024-11-11T20:53:16,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:16,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=B 2024-11-11T20:53:16,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:16,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=C 2024-11-11T20:53:16,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:16,060 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111814407abbe1849b59803b7b3a4c5114d_759cf260483d400f5615c66d4fd85216 is 50, key is test_row_0/A:col10/1731358395995/Put/seqid=0 2024-11-11T20:53:16,076 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:16,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358456068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:16,077 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358396066, max timestamp=1731358396074Num uniques 859; 2024-11-11T20:53:16,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:16,079 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358396066, max timestamp=1731358396073Num uniques 859; 2024-11-11T20:53:16,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:53:16,078 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358396066, max timestamp=1731358396074Num uniques 859; 2024-11-11T20:53:16,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:53:16,081 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:53:16,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-11T20:53:16,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741936_1112 (size=17034) 2024-11-11T20:53:16,103 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:16,110 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111814407abbe1849b59803b7b3a4c5114d_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111814407abbe1849b59803b7b3a4c5114d_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:16,113 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/7fcc2f45594d4483a7d7047369d9acf3, store: [table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:16,119 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/7fcc2f45594d4483a7d7047369d9acf3 is 175, key is test_row_0/A:col10/1731358395995/Put/seqid=0 2024-11-11T20:53:16,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:16,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358456120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:16,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:16,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:16,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358456126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:16,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,131 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:16,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:16,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358456128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:16,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:16,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-11T20:53:16,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:16,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:16,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:16,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:16,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:16,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:16,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358456130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:16,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:16,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:16,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741937_1113 (size=48139) 2024-11-11T20:53:16,142 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/7fcc2f45594d4483a7d7047369d9acf3 2024-11-11T20:53:16,170 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/96ec7315683546aeb248f47d79dfbca1 is 50, key is test_row_0/B:col10/1731358395995/Put/seqid=0 2024-11-11T20:53:16,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741938_1114 (size=12001) 2024-11-11T20:53:16,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/96ec7315683546aeb248f47d79dfbca1 2024-11-11T20:53:16,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:16,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358456185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:16,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,190 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:16,207 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/5b36a1998bcd4514b63fbf1aa05d4e77 is 50, key is test_row_0/C:col10/1731358395995/Put/seqid=0 2024-11-11T20:53:16,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741939_1115 (size=12001) 2024-11-11T20:53:16,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:16,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358456236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:16,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:16,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:16,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:16,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358456236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:16,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358456236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:16,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,248 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:16,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:16,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358456245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:16,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:16,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:16,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-11T20:53:16,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:16,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:16,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:16,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:16,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:16,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:16,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-11T20:53:16,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:16,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358456395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:16,398 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,398 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,398 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:16,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-11T20:53:16,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:16,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:16,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:16,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:16,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:16,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:16,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:16,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358456456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:16,457 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:16,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:16,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358456456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:16,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358456457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:16,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:16,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:16,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:16,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,458 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:16,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358456457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:16,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,458 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:16,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-11T20:53:16,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:16,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:16,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:16,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:16,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:16,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:16,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-11T20:53:16,626 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/5b36a1998bcd4514b63fbf1aa05d4e77 2024-11-11T20:53:16,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/7fcc2f45594d4483a7d7047369d9acf3 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/7fcc2f45594d4483a7d7047369d9acf3 2024-11-11T20:53:16,643 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/7fcc2f45594d4483a7d7047369d9acf3, entries=250, sequenceid=15, filesize=47.0 K 2024-11-11T20:53:16,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/96ec7315683546aeb248f47d79dfbca1 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/96ec7315683546aeb248f47d79dfbca1 2024-11-11T20:53:16,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/96ec7315683546aeb248f47d79dfbca1, entries=150, sequenceid=15, filesize=11.7 K 2024-11-11T20:53:16,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/5b36a1998bcd4514b63fbf1aa05d4e77 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/5b36a1998bcd4514b63fbf1aa05d4e77 2024-11-11T20:53:16,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/5b36a1998bcd4514b63fbf1aa05d4e77, entries=150, sequenceid=15, filesize=11.7 K 2024-11-11T20:53:16,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 759cf260483d400f5615c66d4fd85216 in 673ms, sequenceid=15, compaction requested=false 2024-11-11T20:53:16,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:16,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:16,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 759cf260483d400f5615c66d4fd85216 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-11T20:53:16,720 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=A 2024-11-11T20:53:16,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:16,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=B 2024-11-11T20:53:16,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:16,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=C 2024-11-11T20:53:16,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:16,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111174b163057cd34c198fd96e7a17324fad_759cf260483d400f5615c66d4fd85216 is 50, key is test_row_0/A:col10/1731358396128/Put/seqid=0 2024-11-11T20:53:16,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-11T20:53:16,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:16,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:16,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:16,750 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:16,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:16,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:16,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741940_1116 (size=14594) 2024-11-11T20:53:16,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:16,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358456775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:16,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:16,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358456777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:16,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,781 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:16,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:16,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358456778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:16,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:16,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358456780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:16,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:16,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:16,791 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:16,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358456781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:16,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:16,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:16,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358456886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:16,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:16,890 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:16,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358456886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:16,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,891 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:16,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:16,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358456895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:16,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:16,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358456895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:16,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:16,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358456896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:16,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:16,900 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:16,901 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:16,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-11T20:53:16,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:16,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:16,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:16,903 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:16,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:16,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:17,057 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-11T20:53:17,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:17,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:17,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:17,057 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:17,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:17,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:17,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358457096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:17,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:17,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358457096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:17,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:17,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:17,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358457106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:17,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358457106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:17,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:17,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358457106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:17,107 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:17,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:17,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-11T20:53:17,168 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:17,173 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111174b163057cd34c198fd96e7a17324fad_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111174b163057cd34c198fd96e7a17324fad_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:17,174 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/4e4cef9ed1164174ab1a8cb770757052, store: [table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:17,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/4e4cef9ed1164174ab1a8cb770757052 is 175, key is test_row_0/A:col10/1731358396128/Put/seqid=0 2024-11-11T20:53:17,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741941_1117 (size=39549) 2024-11-11T20:53:17,183 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/4e4cef9ed1164174ab1a8cb770757052 2024-11-11T20:53:17,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/499bfab9ad9f4778bbff5166c98611d6 is 50, key is test_row_0/B:col10/1731358396128/Put/seqid=0 2024-11-11T20:53:17,210 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-11T20:53:17,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:17,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:17,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:17,211 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:17,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:17,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:17,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741942_1118 (size=12001) 2024-11-11T20:53:17,218 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/499bfab9ad9f4778bbff5166c98611d6 2024-11-11T20:53:17,231 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/9a4e5687970747589d2a7e813e558145 is 50, key is test_row_0/C:col10/1731358396128/Put/seqid=0 2024-11-11T20:53:17,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741943_1119 (size=12001) 2024-11-11T20:53:17,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-11T20:53:17,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:17,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:17,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:17,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:17,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:17,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:17,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:17,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:17,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358457405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:17,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358457405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:17,407 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:17,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:17,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:17,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358457416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:17,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:17,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358457416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:17,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,420 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:17,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:17,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358457416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:17,420 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:17,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,421 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:17,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-11T20:53:17,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:17,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:17,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:17,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] handler.RSProcedureHandler(58): pid=33 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:17,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=33 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:17,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=33 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:17,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/9a4e5687970747589d2a7e813e558145 2024-11-11T20:53:17,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/4e4cef9ed1164174ab1a8cb770757052 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/4e4cef9ed1164174ab1a8cb770757052 2024-11-11T20:53:17,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/4e4cef9ed1164174ab1a8cb770757052, entries=200, sequenceid=52, filesize=38.6 K 2024-11-11T20:53:17,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/499bfab9ad9f4778bbff5166c98611d6 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/499bfab9ad9f4778bbff5166c98611d6 2024-11-11T20:53:17,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/499bfab9ad9f4778bbff5166c98611d6, entries=150, sequenceid=52, filesize=11.7 K 2024-11-11T20:53:17,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/9a4e5687970747589d2a7e813e558145 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/9a4e5687970747589d2a7e813e558145 2024-11-11T20:53:17,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/9a4e5687970747589d2a7e813e558145, entries=150, sequenceid=52, filesize=11.7 K 2024-11-11T20:53:17,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=60.38 KB/61830 for 759cf260483d400f5615c66d4fd85216 in 948ms, sequenceid=52, compaction requested=false 2024-11-11T20:53:17,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:17,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=33 2024-11-11T20:53:17,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:17,671 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2902): Flushing 759cf260483d400f5615c66d4fd85216 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T20:53:17,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=A 2024-11-11T20:53:17,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:17,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=B 2024-11-11T20:53:17,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:17,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=C 2024-11-11T20:53:17,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:17,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111accac9117c7b45cc96852bb31756072a_759cf260483d400f5615c66d4fd85216 is 50, key is test_row_0/A:col10/1731358396775/Put/seqid=0 2024-11-11T20:53:17,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741944_1120 (size=12154) 2024-11-11T20:53:17,902 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T20:53:17,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:17,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:17,966 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:17,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358457958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:17,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:17,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358457958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:17,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:17,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358457960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:17,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:17,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358457961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:17,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:17,968 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:17,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358457962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:17,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,969 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:17,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,969 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:17,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:17,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:18,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:18,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358458076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:18,077 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:18,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358458076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:18,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:18,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358458076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:18,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:18,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358458077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:18,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:18,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358458077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:18,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,079 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:18,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,080 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:18,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,080 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:18,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:18,092 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111accac9117c7b45cc96852bb31756072a_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111accac9117c7b45cc96852bb31756072a_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:18,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/84a3fe575a044054967a917cad0ee78c, store: [table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:18,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/84a3fe575a044054967a917cad0ee78c is 175, key is test_row_0/A:col10/1731358396775/Put/seqid=0 2024-11-11T20:53:18,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741945_1121 (size=30955) 2024-11-11T20:53:18,108 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=64, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/84a3fe575a044054967a917cad0ee78c 2024-11-11T20:53:18,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/e4738c4e5868480e9f400032008cc094 is 50, key is test_row_0/B:col10/1731358396775/Put/seqid=0 2024-11-11T20:53:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-11T20:53:18,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741946_1122 (size=12001) 2024-11-11T20:53:18,139 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/e4738c4e5868480e9f400032008cc094 2024-11-11T20:53:18,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/c39e1ddbb0284e9397c08eb63c8840cf is 50, key is test_row_0/C:col10/1731358396775/Put/seqid=0 2024-11-11T20:53:18,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741947_1123 (size=12001) 2024-11-11T20:53:18,182 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=64 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/c39e1ddbb0284e9397c08eb63c8840cf 2024-11-11T20:53:18,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/84a3fe575a044054967a917cad0ee78c as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/84a3fe575a044054967a917cad0ee78c 2024-11-11T20:53:18,201 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/84a3fe575a044054967a917cad0ee78c, entries=150, sequenceid=64, filesize=30.2 K 2024-11-11T20:53:18,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/e4738c4e5868480e9f400032008cc094 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/e4738c4e5868480e9f400032008cc094 2024-11-11T20:53:18,212 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/e4738c4e5868480e9f400032008cc094, entries=150, sequenceid=64, filesize=11.7 K 2024-11-11T20:53:18,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/c39e1ddbb0284e9397c08eb63c8840cf as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/c39e1ddbb0284e9397c08eb63c8840cf 2024-11-11T20:53:18,219 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/c39e1ddbb0284e9397c08eb63c8840cf, entries=150, sequenceid=64, filesize=11.7 K 2024-11-11T20:53:18,220 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 759cf260483d400f5615c66d4fd85216 in 549ms, sequenceid=64, compaction requested=true 2024-11-11T20:53:18,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.HRegion(2603): Flush status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:18,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:18,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-11-11T20:53:18,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-11-11T20:53:18,224 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-11T20:53:18,224 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2420 sec 2024-11-11T20:53:18,226 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=32, table=TestAcidGuarantees in 2.2490 sec 2024-11-11T20:53:18,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:18,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 759cf260483d400f5615c66d4fd85216 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-11T20:53:18,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358397958, max timestamp=1731358398286Num uniques 859; 2024-11-11T20:53:18,290 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:53:18,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358397958, max timestamp=1731358398286Num uniques 859; 2024-11-11T20:53:18,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358397958, max timestamp=1731358398286Num uniques 859; 2024-11-11T20:53:18,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=A 2024-11-11T20:53:18,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:53:18,291 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:53:18,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:18,291 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=B 2024-11-11T20:53:18,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:18,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=C 2024-11-11T20:53:18,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:18,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b23d0a0a05f8405da56beea940a5e7e7_759cf260483d400f5615c66d4fd85216 is 50, key is test_row_0/A:col10/1731358398286/Put/seqid=0 2024-11-11T20:53:18,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741948_1124 (size=12154) 2024-11-11T20:53:18,328 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:18,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1731358398296, max timestamp=1731358398333Num uniques 859; 2024-11-11T20:53:18,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1731358398296, max timestamp=1731358398333Num uniques 859; 2024-11-11T20:53:18,336 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-11T20:53:18,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1731358398296, max timestamp=1731358398333Num uniques 859; 2024-11-11T20:53:18,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-11T20:53:18,338 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b23d0a0a05f8405da56beea940a5e7e7_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b23d0a0a05f8405da56beea940a5e7e7_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:18,337 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-11T20:53:18,340 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/2854ca14b9d54f6193374e9b3c5fc61f, store: [table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:18,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/2854ca14b9d54f6193374e9b3c5fc61f is 175, key is test_row_0/A:col10/1731358398286/Put/seqid=0 2024-11-11T20:53:18,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741949_1125 (size=30955) 2024-11-11T20:53:18,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:18,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358458357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:18,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358458357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:18,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358458360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:18,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,362 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:18,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,362 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:18,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,363 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:18,372 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:18,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358458361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:18,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:18,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:18,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358458363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:18,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,377 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:18,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358458466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:18,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:18,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358458466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:18,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:18,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358458466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:18,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:18,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:18,482 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:18,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358458476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:18,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,482 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:18,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:18,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358458486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:18,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:18,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:18,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358458676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:18,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358458676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:18,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358458676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,680 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:18,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:18,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358458686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:18,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:18,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:18,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358458695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:18,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,701 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:18,749 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=87, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/2854ca14b9d54f6193374e9b3c5fc61f 2024-11-11T20:53:18,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/5a92cdb551d34f1aab1e13e7e0d33ccd is 50, key is test_row_0/B:col10/1731358398286/Put/seqid=0 2024-11-11T20:53:18,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741950_1126 (size=12001) 2024-11-11T20:53:18,987 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358458986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:18,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358458986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:18,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:18,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358458986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:18,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:18,989 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:19,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358459005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:19,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:19,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:19,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358459005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:19,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:19,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:19,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:19,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:19,009 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:19,162 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/5a92cdb551d34f1aab1e13e7e0d33ccd 2024-11-11T20:53:19,172 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/dfd9e5650e724365be02cd64b027e39b is 50, key is test_row_0/C:col10/1731358398286/Put/seqid=0 2024-11-11T20:53:19,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741951_1127 (size=12001) 2024-11-11T20:53:19,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/dfd9e5650e724365be02cd64b027e39b 2024-11-11T20:53:19,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/2854ca14b9d54f6193374e9b3c5fc61f as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/2854ca14b9d54f6193374e9b3c5fc61f 2024-11-11T20:53:19,193 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/2854ca14b9d54f6193374e9b3c5fc61f, entries=150, sequenceid=87, filesize=30.2 K 2024-11-11T20:53:19,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/5a92cdb551d34f1aab1e13e7e0d33ccd as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/5a92cdb551d34f1aab1e13e7e0d33ccd 2024-11-11T20:53:19,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/5a92cdb551d34f1aab1e13e7e0d33ccd, entries=150, sequenceid=87, filesize=11.7 K 2024-11-11T20:53:19,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/dfd9e5650e724365be02cd64b027e39b as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/dfd9e5650e724365be02cd64b027e39b 2024-11-11T20:53:19,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/dfd9e5650e724365be02cd64b027e39b, entries=150, sequenceid=87, filesize=11.7 K 2024-11-11T20:53:19,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.03 KB/108576, currentSize=154.31 KB/158010 for 759cf260483d400f5615c66d4fd85216 in 921ms, sequenceid=87, compaction requested=true 2024-11-11T20:53:19,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:19,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 759cf260483d400f5615c66d4fd85216:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:53:19,211 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:53:19,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:19,211 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:53:19,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 759cf260483d400f5615c66d4fd85216:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:53:19,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:19,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 759cf260483d400f5615c66d4fd85216:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:53:19,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:19,213 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:53:19,213 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 759cf260483d400f5615c66d4fd85216/B is initiating minor compaction (all files) 2024-11-11T20:53:19,213 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 759cf260483d400f5615c66d4fd85216/B in TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:19,213 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/96ec7315683546aeb248f47d79dfbca1, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/499bfab9ad9f4778bbff5166c98611d6, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/e4738c4e5868480e9f400032008cc094, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/5a92cdb551d34f1aab1e13e7e0d33ccd] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp, totalSize=46.9 K 2024-11-11T20:53:19,214 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149598 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:53:19,214 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 96ec7315683546aeb248f47d79dfbca1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731358395988 2024-11-11T20:53:19,214 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 759cf260483d400f5615c66d4fd85216/A is initiating minor compaction (all files) 2024-11-11T20:53:19,214 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 759cf260483d400f5615c66d4fd85216/A in TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:19,214 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/7fcc2f45594d4483a7d7047369d9acf3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/4e4cef9ed1164174ab1a8cb770757052, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/84a3fe575a044054967a917cad0ee78c, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/2854ca14b9d54f6193374e9b3c5fc61f] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp, totalSize=146.1 K 2024-11-11T20:53:19,214 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:19,214 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. files: [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/7fcc2f45594d4483a7d7047369d9acf3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/4e4cef9ed1164174ab1a8cb770757052, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/84a3fe575a044054967a917cad0ee78c, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/2854ca14b9d54f6193374e9b3c5fc61f] 2024-11-11T20:53:19,215 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 499bfab9ad9f4778bbff5166c98611d6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731358396101 2024-11-11T20:53:19,215 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fcc2f45594d4483a7d7047369d9acf3, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731358395987 2024-11-11T20:53:19,216 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting e4738c4e5868480e9f400032008cc094, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731358396775 2024-11-11T20:53:19,216 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e4cef9ed1164174ab1a8cb770757052, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731358396101 2024-11-11T20:53:19,216 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a92cdb551d34f1aab1e13e7e0d33ccd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731358397960 2024-11-11T20:53:19,216 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84a3fe575a044054967a917cad0ee78c, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731358396775 2024-11-11T20:53:19,217 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2854ca14b9d54f6193374e9b3c5fc61f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731358397960 2024-11-11T20:53:19,230 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 759cf260483d400f5615c66d4fd85216#B#compaction#108 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:19,231 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/fe04eb1c133f4940b361bd7e7dba1f2b is 50, key is test_row_0/B:col10/1731358398286/Put/seqid=0 2024-11-11T20:53:19,234 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:19,246 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411114dd50ae706ad4c49be84b64596531711_759cf260483d400f5615c66d4fd85216 store=[table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:19,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741952_1128 (size=12139) 2024-11-11T20:53:19,256 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411114dd50ae706ad4c49be84b64596531711_759cf260483d400f5615c66d4fd85216, store=[table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:19,257 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114dd50ae706ad4c49be84b64596531711_759cf260483d400f5615c66d4fd85216 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:19,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741953_1129 (size=4469) 2024-11-11T20:53:19,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:19,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 759cf260483d400f5615c66d4fd85216 3/3 column families, dataSize=161.02 KB heapSize=436.55 KB 2024-11-11T20:53:19,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=A 2024-11-11T20:53:19,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:19,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=B 2024-11-11T20:53:19,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:19,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=C 2024-11-11T20:53:19,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:19,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:19,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358459504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:19,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:19,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:19,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358459507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:19,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:19,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:19,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:19,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358459510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:19,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:19,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:19,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:19,511 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d84a1238b55247c3a6c58e7b1ada70ab_759cf260483d400f5615c66d4fd85216 is 50, key is test_row_0/A:col10/1731358398355/Put/seqid=0 2024-11-11T20:53:19,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:19,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:19,512 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:19,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:19,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358459516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:19,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:19,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:19,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:19,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358459516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:19,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:19,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:19,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:19,517 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:19,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741954_1130 (size=14594) 2024-11-11T20:53:19,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:19,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358459616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:19,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:19,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:19,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:19,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358459616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:19,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:19,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:19,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:19,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:19,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:19,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358459616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:19,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:19,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:19,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:19,656 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/fe04eb1c133f4940b361bd7e7dba1f2b as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/fe04eb1c133f4940b361bd7e7dba1f2b 2024-11-11T20:53:19,664 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 759cf260483d400f5615c66d4fd85216/B of 759cf260483d400f5615c66d4fd85216 into fe04eb1c133f4940b361bd7e7dba1f2b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:19,664 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:19,664 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., storeName=759cf260483d400f5615c66d4fd85216/B, priority=12, startTime=1731358399211; duration=0sec 2024-11-11T20:53:19,664 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:19,664 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 759cf260483d400f5615c66d4fd85216:B 2024-11-11T20:53:19,665 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 759cf260483d400f5615c66d4fd85216#A#compaction#109 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:19,665 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:53:19,666 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/96247eb32be84156a54279415983741b is 175, key is test_row_0/A:col10/1731358398286/Put/seqid=0 2024-11-11T20:53:19,669 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:53:19,669 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 759cf260483d400f5615c66d4fd85216/C is initiating minor compaction (all files) 2024-11-11T20:53:19,669 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 759cf260483d400f5615c66d4fd85216/C in TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:19,669 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/5b36a1998bcd4514b63fbf1aa05d4e77, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/9a4e5687970747589d2a7e813e558145, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/c39e1ddbb0284e9397c08eb63c8840cf, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/dfd9e5650e724365be02cd64b027e39b] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp, totalSize=46.9 K 2024-11-11T20:53:19,670 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b36a1998bcd4514b63fbf1aa05d4e77, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731358395988 2024-11-11T20:53:19,670 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a4e5687970747589d2a7e813e558145, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731358396101 2024-11-11T20:53:19,670 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting c39e1ddbb0284e9397c08eb63c8840cf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=64, earliestPutTs=1731358396775 2024-11-11T20:53:19,671 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting dfd9e5650e724365be02cd64b027e39b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731358397960 2024-11-11T20:53:19,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741955_1131 (size=31093) 2024-11-11T20:53:19,685 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 759cf260483d400f5615c66d4fd85216#C#compaction#111 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:19,686 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/c8fc8c4334bf49db93b5c460315e74f4 is 50, key is test_row_0/C:col10/1731358398286/Put/seqid=0 2024-11-11T20:53:19,698 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/96247eb32be84156a54279415983741b as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/96247eb32be84156a54279415983741b 2024-11-11T20:53:19,705 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 759cf260483d400f5615c66d4fd85216/A of 759cf260483d400f5615c66d4fd85216 into 96247eb32be84156a54279415983741b(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:19,705 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:19,705 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., storeName=759cf260483d400f5615c66d4fd85216/A, priority=12, startTime=1731358399211; duration=0sec 2024-11-11T20:53:19,705 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:19,705 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 759cf260483d400f5615c66d4fd85216:A 2024-11-11T20:53:19,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741956_1132 (size=12139) 2024-11-11T20:53:19,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:19,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:19,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358459826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:19,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:19,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358459826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:19,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358459826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:19,927 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:19,932 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d84a1238b55247c3a6c58e7b1ada70ab_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d84a1238b55247c3a6c58e7b1ada70ab_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:19,933 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/5d8dbde7f6134230b786b825663af49c, store: [table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:19,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/5d8dbde7f6134230b786b825663af49c is 175, key is test_row_0/A:col10/1731358398355/Put/seqid=0 2024-11-11T20:53:19,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741957_1133 (size=39549) 2024-11-11T20:53:20,112 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/c8fc8c4334bf49db93b5c460315e74f4 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/c8fc8c4334bf49db93b5c460315e74f4 2024-11-11T20:53:20,118 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 759cf260483d400f5615c66d4fd85216/C of 759cf260483d400f5615c66d4fd85216 into c8fc8c4334bf49db93b5c460315e74f4(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:20,118 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:20,118 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., storeName=759cf260483d400f5615c66d4fd85216/C, priority=12, startTime=1731358399211; duration=0sec 2024-11-11T20:53:20,118 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:20,118 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 759cf260483d400f5615c66d4fd85216:C 2024-11-11T20:53:20,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:20,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358460135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:20,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:20,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358460135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:20,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:20,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:20,137 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:20,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:20,137 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:20,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:20,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:20,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358460136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:20,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:20,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:20,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:20,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-11T20:53:20,146 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:53:20,147 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:53:20,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2024-11-11T20:53:20,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-11T20:53:20,149 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:53:20,149 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:53:20,149 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:53:20,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-11T20:53:20,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-11T20:53:20,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:20,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:20,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:20,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:20,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:20,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:20,337 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/5d8dbde7f6134230b786b825663af49c 2024-11-11T20:53:20,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/b29051c5cbce497b892e15a932019c39 is 50, key is test_row_0/B:col10/1731358398355/Put/seqid=0 2024-11-11T20:53:20,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741958_1134 (size=12001) 2024-11-11T20:53:20,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-11T20:53:20,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:20,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:20,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:20,454 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:20,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:20,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:20,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-11T20:53:20,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:20,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358460526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:20,528 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_0' in 759cf260483d400f5615c66d4fd85216 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:20,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:20,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:20,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:20,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:20,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358460535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:20,537 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in 759cf260483d400f5615c66d4fd85216 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:20,538 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:20,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-11T20:53:20,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:20,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:20,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:20,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:20,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:20,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:20,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:20,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:20,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358460645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:20,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358460645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:20,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:20,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358460645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:20,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:20,750 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/b29051c5cbce497b892e15a932019c39 2024-11-11T20:53:20,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-11T20:53:20,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:20,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:20,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:20,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:20,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:20,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/bc14a3c8775045cf80c31aaac545c332 is 50, key is test_row_0/C:col10/1731358398355/Put/seqid=0 2024-11-11T20:53:20,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:20,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741959_1135 (size=12001) 2024-11-11T20:53:20,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-11T20:53:20,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-11T20:53:20,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:20,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:20,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:20,914 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:20,914 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:20,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:21,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-11T20:53:21,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:21,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:21,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:21,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:21,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:21,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:21,172 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/bc14a3c8775045cf80c31aaac545c332 2024-11-11T20:53:21,177 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/5d8dbde7f6134230b786b825663af49c as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/5d8dbde7f6134230b786b825663af49c 2024-11-11T20:53:21,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/5d8dbde7f6134230b786b825663af49c, entries=200, sequenceid=122, filesize=38.6 K 2024-11-11T20:53:21,184 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/b29051c5cbce497b892e15a932019c39 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/b29051c5cbce497b892e15a932019c39 2024-11-11T20:53:21,190 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/b29051c5cbce497b892e15a932019c39, entries=150, sequenceid=122, filesize=11.7 K 2024-11-11T20:53:21,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/bc14a3c8775045cf80c31aaac545c332 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/bc14a3c8775045cf80c31aaac545c332 2024-11-11T20:53:21,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/bc14a3c8775045cf80c31aaac545c332, entries=150, sequenceid=122, filesize=11.7 K 2024-11-11T20:53:21,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~436.50 KB/446976, currentSize=26.84 KB/27480 for 759cf260483d400f5615c66d4fd85216 in 1702ms, sequenceid=122, compaction requested=false 2024-11-11T20:53:21,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:21,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-11T20:53:21,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:21,220 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing 759cf260483d400f5615c66d4fd85216 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-11T20:53:21,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=A 2024-11-11T20:53:21,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:21,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=B 2024-11-11T20:53:21,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:21,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=C 2024-11-11T20:53:21,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:21,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111452159d37c374984b3072786f5bac237_759cf260483d400f5615c66d4fd85216 is 50, key is test_row_0/A:col10/1731358399505/Put/seqid=0 2024-11-11T20:53:21,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741960_1136 (size=12154) 2024-11-11T20:53:21,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-11T20:53:21,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:21,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:21,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:21,671 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111452159d37c374984b3072786f5bac237_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111452159d37c374984b3072786f5bac237_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:21,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/047f0ea3ecc648f09aa8c3832a4cb76a, store: [table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:21,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/047f0ea3ecc648f09aa8c3832a4cb76a is 175, key is test_row_0/A:col10/1731358399505/Put/seqid=0 2024-11-11T20:53:21,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741961_1137 (size=30955) 2024-11-11T20:53:21,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731358401672, max timestamp=1731358401686Num uniques 859; 2024-11-11T20:53:21,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731358401672, max timestamp=1731358401686Num uniques 859; 2024-11-11T20:53:21,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T20:53:21,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731358401672, max timestamp=1731358401686Num uniques 859; 2024-11-11T20:53:21,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T20:53:21,689 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T20:53:21,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731358401706, max timestamp=1731358401725Num uniques 1118; 2024-11-11T20:53:21,728 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=14558, suffix heap size=144240,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=318 2024-11-11T20:53:21,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731358401706, max timestamp=1731358401725Num uniques 1118; 2024-11-11T20:53:21,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=14558, suffix heap size=144240,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=318 2024-11-11T20:53:21,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731358401706, max timestamp=1731358401725Num uniques 1118; 2024-11-11T20:53:21,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=14558, suffix heap size=144240,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=318 2024-11-11T20:53:21,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:21,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358461765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:21,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:21,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358461766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:21,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:21,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:21,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:21,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:21,771 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:21,771 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:21,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1731358401751, max timestamp=1731358401768Num uniques 1177; 2024-11-11T20:53:21,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1731358401751, max timestamp=1731358401768Num uniques 1177; 2024-11-11T20:53:21,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=327, cellSize=14972, totalHeapSize=34248, min timestamp=1731358401751, max timestamp=1731358401768Num uniques 1177; 2024-11-11T20:53:21,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=14972, suffix heap size=150376,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=327 2024-11-11T20:53:21,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=14972, suffix heap size=150376,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=327 2024-11-11T20:53:21,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=53902, new segment data size=14972, suffix heap size=150376,new segment heap size=34248 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1177, new segment cells count=327 2024-11-11T20:53:21,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731358401826, max timestamp=1731358401855Num uniques 1186; 2024-11-11T20:53:21,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731358401826, max timestamp=1731358401855Num uniques 1186; 2024-11-11T20:53:21,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=10806, suffix heap size=151312,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=236 2024-11-11T20:53:21,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=10806, suffix heap size=151312,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=236 2024-11-11T20:53:21,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731358401826, max timestamp=1731358401855Num uniques 1186; 2024-11-11T20:53:21,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=54316, new segment data size=10806, suffix heap size=151312,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1186, new segment cells count=236 2024-11-11T20:53:21,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1731358401882, max timestamp=1731358401903Num uniques 1095; 2024-11-11T20:53:21,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=18090, suffix heap size=141848,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=395 2024-11-11T20:53:21,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1731358401882, max timestamp=1731358401903Num uniques 1095; 2024-11-11T20:53:21,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=395, cellSize=18090, totalHeapSize=41320, min timestamp=1731358401882, max timestamp=1731358401903Num uniques 1095; 2024-11-11T20:53:21,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=18090, suffix heap size=141848,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=395 2024-11-11T20:53:21,907 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=18090, suffix heap size=141848,new segment heap size=41320 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=395 2024-11-11T20:53:21,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:21,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358461933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:21,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:21,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:21,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:21,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:21,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358461939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:21,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:21,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:21,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358461939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:21,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:21,944 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:21,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:21,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:21,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:22,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358462045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:22,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358462045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:22,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:22,053 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:22,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358462046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:22,054 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:22,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:22,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:22,054 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:22,077 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/047f0ea3ecc648f09aa8c3832a4cb76a 2024-11-11T20:53:22,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/3c441bac779c4f6c94fc19d5c932df60 is 50, key is test_row_0/B:col10/1731358399505/Put/seqid=0 2024-11-11T20:53:22,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741962_1138 (size=12001) 2024-11-11T20:53:22,111 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/3c441bac779c4f6c94fc19d5c932df60 2024-11-11T20:53:22,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/73bc5d506f2141ee82825bd9a3f5b0b5 is 50, key is test_row_0/C:col10/1731358399505/Put/seqid=0 2024-11-11T20:53:22,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741963_1139 (size=12001) 2024-11-11T20:53:22,168 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/73bc5d506f2141ee82825bd9a3f5b0b5 2024-11-11T20:53:22,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/047f0ea3ecc648f09aa8c3832a4cb76a as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/047f0ea3ecc648f09aa8c3832a4cb76a 2024-11-11T20:53:22,184 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/047f0ea3ecc648f09aa8c3832a4cb76a, entries=150, sequenceid=132, filesize=30.2 K 2024-11-11T20:53:22,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/3c441bac779c4f6c94fc19d5c932df60 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/3c441bac779c4f6c94fc19d5c932df60 2024-11-11T20:53:22,201 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/3c441bac779c4f6c94fc19d5c932df60, entries=150, sequenceid=132, filesize=11.7 K 2024-11-11T20:53:22,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/73bc5d506f2141ee82825bd9a3f5b0b5 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/73bc5d506f2141ee82825bd9a3f5b0b5 2024-11-11T20:53:22,212 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/73bc5d506f2141ee82825bd9a3f5b0b5, entries=150, sequenceid=132, filesize=11.7 K 2024-11-11T20:53:22,213 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=167.72 KB/171750 for 759cf260483d400f5615c66d4fd85216 in 993ms, sequenceid=132, compaction requested=true 2024-11-11T20:53:22,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:22,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:22,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-11T20:53:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-11T20:53:22,226 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-11T20:53:22,226 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0670 sec 2024-11-11T20:53:22,232 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 2.0810 sec 2024-11-11T20:53:22,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731358401931, max timestamp=1731358402257Num uniques 1254; 2024-11-11T20:53:22,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731358401931, max timestamp=1731358402257Num uniques 1254; 2024-11-11T20:53:22,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=57430, new segment data size=7050, suffix heap size=158384,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1254, new segment cells count=154 2024-11-11T20:53:22,261 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731358401931, max timestamp=1731358402257Num uniques 1254; 2024-11-11T20:53:22,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:22,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=57430, new segment data size=7050, suffix heap size=158384,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1254, new segment cells count=154 2024-11-11T20:53:22,262 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=57430, new segment data size=7050, suffix heap size=158384,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1254, new segment cells count=154 2024-11-11T20:53:22,262 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 759cf260483d400f5615c66d4fd85216 3/3 column families, dataSize=33.54 KB heapSize=86.63 KB 2024-11-11T20:53:22,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=A 2024-11-11T20:53:22,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:22,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=B 2024-11-11T20:53:22,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:22,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=C 2024-11-11T20:53:22,262 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:22,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111589d170cdc354022ab1dcf00d6f3968b_759cf260483d400f5615c66d4fd85216 is 50, key is test_row_0/A:col10/1731358401932/Put/seqid=0 2024-11-11T20:53:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-11T20:53:22,306 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:53:22,307 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:53:22,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-11T20:53:22,310 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:53:22,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-11T20:53:22,313 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:53:22,313 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:53:22,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741964_1140 (size=17284) 2024-11-11T20:53:22,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358402297, max timestamp=1731358402329Num uniques 859; 2024-11-11T20:53:22,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:53:22,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358402297, max timestamp=1731358402329Num uniques 859; 2024-11-11T20:53:22,335 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:53:22,336 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:22,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358402297, max timestamp=1731358402331Num uniques 859; 2024-11-11T20:53:22,338 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:53:22,344 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111589d170cdc354022ab1dcf00d6f3968b_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111589d170cdc354022ab1dcf00d6f3968b_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:22,348 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/116ff75ad6e94b79983223b36092bfce, store: [table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:22,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/116ff75ad6e94b79983223b36092bfce is 175, key is test_row_0/A:col10/1731358401932/Put/seqid=0 2024-11-11T20:53:22,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741965_1141 (size=48389) 2024-11-11T20:53:22,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-11T20:53:22,416 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731358402384, max timestamp=1731358402413Num uniques 1018; 2024-11-11T20:53:22,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731358402384, max timestamp=1731358402413Num uniques 1018; 2024-11-11T20:53:22,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-11T20:53:22,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731358402384, max timestamp=1731358402413Num uniques 1018; 2024-11-11T20:53:22,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-11T20:53:22,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-11T20:53:22,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1731358402431, max timestamp=1731358402461Num uniques 1077; 2024-11-11T20:53:22,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=17262, suffix heap size=139976,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=377 2024-11-11T20:53:22,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1731358402431, max timestamp=1731358402461Num uniques 1077; 2024-11-11T20:53:22,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=17262, suffix heap size=139976,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=377 2024-11-11T20:53:22,465 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=377, cellSize=17262, totalHeapSize=39448, min timestamp=1731358402431, max timestamp=1731358402461Num uniques 1077; 2024-11-11T20:53:22,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=17262, suffix heap size=139976,new segment heap size=39448 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=377 2024-11-11T20:53:22,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T20:53:22,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:22,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:22,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:22,468 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:22,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:22,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358462505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:22,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:22,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:22,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:22,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358462506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:22,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:22,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:22,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358462506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:22,515 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:22,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:22,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:22,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:22,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358462535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:22,539 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_0' in 759cf260483d400f5615c66d4fd85216 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:22,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:22,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:22,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:22,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:22,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358462545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:22,550 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in 759cf260483d400f5615c66d4fd85216 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:22,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:22,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:22,551 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:22,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:22,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358462615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:22,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:22,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:22,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:22,620 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T20:53:22,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:22,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:22,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:22,621 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:22,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:22,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:22,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-11T20:53:22,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:22,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358462625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:22,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:22,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:22,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358462626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:22,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:22,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T20:53:22,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:22,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:22,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:22,774 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:22,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:22,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:22,798 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=240, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/116ff75ad6e94b79983223b36092bfce 2024-11-11T20:53:22,806 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/f612d51028204073a0dae33a07b11303 is 50, key is test_row_0/B:col10/1731358401932/Put/seqid=0 2024-11-11T20:53:22,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741966_1142 (size=12151) 2024-11-11T20:53:22,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:22,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358462825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:22,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:22,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358462835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:22,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:22,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:22,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:22,840 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:22,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358462836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:22,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:22,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:22,841 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:22,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T20:53:22,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:22,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:22,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:22,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:22,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:22,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-11T20:53:23,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T20:53:23,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:23,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:23,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:23,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:23,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:23,136 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358463135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:23,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:23,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:23,137 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:23,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358463145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:23,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:23,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:23,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:23,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:23,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358463145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:23,149 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:23,149 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:23,149 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:23,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/f612d51028204073a0dae33a07b11303 2024-11-11T20:53:23,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T20:53:23,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:23,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:23,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:23,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:23,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:23,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/383d1611c40746d39e169fc5427bfac1 is 50, key is test_row_0/C:col10/1731358401932/Put/seqid=0 2024-11-11T20:53:23,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:23,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741967_1143 (size=12151) 2024-11-11T20:53:23,243 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=240 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/383d1611c40746d39e169fc5427bfac1 2024-11-11T20:53:23,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/116ff75ad6e94b79983223b36092bfce as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/116ff75ad6e94b79983223b36092bfce 2024-11-11T20:53:23,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/116ff75ad6e94b79983223b36092bfce, entries=250, sequenceid=240, filesize=47.3 K 2024-11-11T20:53:23,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/f612d51028204073a0dae33a07b11303 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/f612d51028204073a0dae33a07b11303 2024-11-11T20:53:23,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/f612d51028204073a0dae33a07b11303, entries=150, sequenceid=240, filesize=11.9 K 2024-11-11T20:53:23,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/383d1611c40746d39e169fc5427bfac1 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/383d1611c40746d39e169fc5427bfac1 2024-11-11T20:53:23,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/383d1611c40746d39e169fc5427bfac1, entries=150, sequenceid=240, filesize=11.9 K 2024-11-11T20:53:23,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.58 KB/88656, currentSize=161.02 KB/164880 for 759cf260483d400f5615c66d4fd85216 in 1032ms, sequenceid=240, compaction requested=true 2024-11-11T20:53:23,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:23,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 759cf260483d400f5615c66d4fd85216:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:53:23,295 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:23,295 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:53:23,295 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:53:23,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 759cf260483d400f5615c66d4fd85216:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:53:23,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:23,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 759cf260483d400f5615c66d4fd85216:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:53:23,296 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:23,298 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 149986 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:53:23,298 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:53:23,298 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 759cf260483d400f5615c66d4fd85216/B is initiating minor compaction (all files) 2024-11-11T20:53:23,298 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 759cf260483d400f5615c66d4fd85216/A is initiating minor compaction (all files) 2024-11-11T20:53:23,299 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 759cf260483d400f5615c66d4fd85216/A in TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:23,299 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 759cf260483d400f5615c66d4fd85216/B in TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:23,299 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/fe04eb1c133f4940b361bd7e7dba1f2b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/b29051c5cbce497b892e15a932019c39, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/3c441bac779c4f6c94fc19d5c932df60, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/f612d51028204073a0dae33a07b11303] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp, totalSize=47.2 K 2024-11-11T20:53:23,299 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/96247eb32be84156a54279415983741b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/5d8dbde7f6134230b786b825663af49c, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/047f0ea3ecc648f09aa8c3832a4cb76a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/116ff75ad6e94b79983223b36092bfce] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp, totalSize=146.5 K 2024-11-11T20:53:23,299 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:23,299 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. files: [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/96247eb32be84156a54279415983741b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/5d8dbde7f6134230b786b825663af49c, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/047f0ea3ecc648f09aa8c3832a4cb76a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/116ff75ad6e94b79983223b36092bfce] 2024-11-11T20:53:23,300 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting fe04eb1c133f4940b361bd7e7dba1f2b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731358397960 2024-11-11T20:53:23,300 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96247eb32be84156a54279415983741b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731358397960 2024-11-11T20:53:23,300 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting b29051c5cbce497b892e15a932019c39, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731358398355 2024-11-11T20:53:23,300 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d8dbde7f6134230b786b825663af49c, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731358398355 2024-11-11T20:53:23,300 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c441bac779c4f6c94fc19d5c932df60, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731358399498 2024-11-11T20:53:23,301 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 047f0ea3ecc648f09aa8c3832a4cb76a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731358399498 2024-11-11T20:53:23,301 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting f612d51028204073a0dae33a07b11303, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731358401931 2024-11-11T20:53:23,301 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 116ff75ad6e94b79983223b36092bfce, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731358401931 2024-11-11T20:53:23,320 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:23,340 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 759cf260483d400f5615c66d4fd85216#B#compaction#121 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:23,341 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/e646d233d4ef44d7abcb03b76eaab84a is 50, key is test_row_0/B:col10/1731358401932/Put/seqid=0 2024-11-11T20:53:23,345 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111aa73d8e5cfc04bccb186e9bd167a0866_759cf260483d400f5615c66d4fd85216 store=[table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:23,348 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111aa73d8e5cfc04bccb186e9bd167a0866_759cf260483d400f5615c66d4fd85216, store=[table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:23,348 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111aa73d8e5cfc04bccb186e9bd167a0866_759cf260483d400f5615c66d4fd85216 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:23,386 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-11T20:53:23,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:23,387 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 759cf260483d400f5615c66d4fd85216 3/3 column families, dataSize=161.02 KB heapSize=444.23 KB 2024-11-11T20:53:23,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=A 2024-11-11T20:53:23,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:23,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=B 2024-11-11T20:53:23,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:23,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=C 2024-11-11T20:53:23,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:23,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741968_1144 (size=12425) 2024-11-11T20:53:23,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741969_1145 (size=4469) 2024-11-11T20:53:23,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111837775e91d9d4118b2946a29e88f31f1_759cf260483d400f5615c66d4fd85216 is 50, key is test_row_0/A:col10/1731358402504/Put/seqid=0 2024-11-11T20:53:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-11T20:53:23,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741970_1146 (size=12454) 2024-11-11T20:53:23,467 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:23,473 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111837775e91d9d4118b2946a29e88f31f1_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111837775e91d9d4118b2946a29e88f31f1_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:23,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/03e6fcd7cf864f128c15c541b4e393f9, store: [table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:23,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/03e6fcd7cf864f128c15c541b4e393f9 is 175, key is test_row_0/A:col10/1731358402504/Put/seqid=0 2024-11-11T20:53:23,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741971_1147 (size=31255) 2024-11-11T20:53:23,499 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=311, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/03e6fcd7cf864f128c15c541b4e393f9 2024-11-11T20:53:23,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/e16338392d45432b853e94df2a3cf995 is 50, key is test_row_0/B:col10/1731358402504/Put/seqid=0 2024-11-11T20:53:23,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741972_1148 (size=12301) 2024-11-11T20:53:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:23,660 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:23,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358463673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:23,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358463673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:23,688 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:23,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:23,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:23,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:23,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:23,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358463676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:23,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:23,798 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/e646d233d4ef44d7abcb03b76eaab84a as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/e646d233d4ef44d7abcb03b76eaab84a 2024-11-11T20:53:23,800 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 759cf260483d400f5615c66d4fd85216#A#compaction#120 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:23,801 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/36926a235b224be3b0223482f1d86b65 is 175, key is test_row_0/A:col10/1731358401932/Put/seqid=0 2024-11-11T20:53:23,801 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:23,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358463795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:23,801 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:23,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:23,802 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:23,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:23,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358463795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:23,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:23,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:23,803 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:23,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:23,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358463795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:23,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:23,811 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 759cf260483d400f5615c66d4fd85216/B of 759cf260483d400f5615c66d4fd85216 into e646d233d4ef44d7abcb03b76eaab84a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:23,811 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:23,811 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., storeName=759cf260483d400f5615c66d4fd85216/B, priority=12, startTime=1731358403295; duration=0sec 2024-11-11T20:53:23,811 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:23,811 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 759cf260483d400f5615c66d4fd85216:B 2024-11-11T20:53:23,811 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:53:23,814 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:53:23,815 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 759cf260483d400f5615c66d4fd85216/C is initiating minor compaction (all files) 2024-11-11T20:53:23,815 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 759cf260483d400f5615c66d4fd85216/C in TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:23,815 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/c8fc8c4334bf49db93b5c460315e74f4, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/bc14a3c8775045cf80c31aaac545c332, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/73bc5d506f2141ee82825bd9a3f5b0b5, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/383d1611c40746d39e169fc5427bfac1] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp, totalSize=47.2 K 2024-11-11T20:53:23,816 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting c8fc8c4334bf49db93b5c460315e74f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1731358397960 2024-11-11T20:53:23,816 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting bc14a3c8775045cf80c31aaac545c332, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731358398355 2024-11-11T20:53:23,820 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 73bc5d506f2141ee82825bd9a3f5b0b5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731358399498 2024-11-11T20:53:23,820 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 383d1611c40746d39e169fc5427bfac1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731358401931 2024-11-11T20:53:23,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741973_1149 (size=31379) 2024-11-11T20:53:23,856 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 759cf260483d400f5615c66d4fd85216#C#compaction#124 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:23,857 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/7125a843d7554e5ab1c3e472ae1a3589 is 50, key is test_row_0/C:col10/1731358401932/Put/seqid=0 2024-11-11T20:53:23,858 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/36926a235b224be3b0223482f1d86b65 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/36926a235b224be3b0223482f1d86b65 2024-11-11T20:53:23,873 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 759cf260483d400f5615c66d4fd85216/A of 759cf260483d400f5615c66d4fd85216 into 36926a235b224be3b0223482f1d86b65(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:23,873 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:23,873 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., storeName=759cf260483d400f5615c66d4fd85216/A, priority=12, startTime=1731358403294; duration=0sec 2024-11-11T20:53:23,873 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:23,873 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 759cf260483d400f5615c66d4fd85216:A 2024-11-11T20:53:23,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741974_1150 (size=12425) 2024-11-11T20:53:23,913 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/7125a843d7554e5ab1c3e472ae1a3589 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/7125a843d7554e5ab1c3e472ae1a3589 2024-11-11T20:53:23,929 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 759cf260483d400f5615c66d4fd85216/C of 759cf260483d400f5615c66d4fd85216 into 7125a843d7554e5ab1c3e472ae1a3589(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:23,929 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:23,929 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., storeName=759cf260483d400f5615c66d4fd85216/C, priority=12, startTime=1731358403296; duration=0sec 2024-11-11T20:53:23,929 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:23,929 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 759cf260483d400f5615c66d4fd85216:C 2024-11-11T20:53:23,937 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/e16338392d45432b853e94df2a3cf995 2024-11-11T20:53:23,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/9fc8a1dacfe04b038274bb52be949147 is 50, key is test_row_0/C:col10/1731358402504/Put/seqid=0 2024-11-11T20:53:23,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741975_1151 (size=12301) 2024-11-11T20:53:24,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358464006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:24,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:24,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:24,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358464006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:24,010 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:24,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:24,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358464015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:24,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:24,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:24,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358464316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:24,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:24,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358464316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:24,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:24,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:24,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:24,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:24,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:24,321 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:24,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:24,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358464325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:24,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:24,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:24,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:24,371 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/9fc8a1dacfe04b038274bb52be949147 2024-11-11T20:53:24,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/03e6fcd7cf864f128c15c541b4e393f9 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/03e6fcd7cf864f128c15c541b4e393f9 2024-11-11T20:53:24,395 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/03e6fcd7cf864f128c15c541b4e393f9, entries=150, sequenceid=311, filesize=30.5 K 2024-11-11T20:53:24,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/e16338392d45432b853e94df2a3cf995 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/e16338392d45432b853e94df2a3cf995 2024-11-11T20:53:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,413 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/e16338392d45432b853e94df2a3cf995, entries=150, sequenceid=311, filesize=12.0 K 2024-11-11T20:53:24,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/9fc8a1dacfe04b038274bb52be949147 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/9fc8a1dacfe04b038274bb52be949147 2024-11-11T20:53:24,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-11T20:53:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,464 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/9fc8a1dacfe04b038274bb52be949147, entries=150, sequenceid=311, filesize=12.0 K 2024-11-11T20:53:24,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,465 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~444.19 KB/454848, currentSize=26.84 KB/27480 for 759cf260483d400f5615c66d4fd85216 in 1079ms, sequenceid=311, compaction requested=false 2024-11-11T20:53:24,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:24,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:24,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-11T20:53:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-11T20:53:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-11T20:53:24,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1540 sec 2024-11-11T20:53:24,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,475 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 2.1650 sec 2024-11-11T20:53:24,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:24,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 759cf260483d400f5615c66d4fd85216 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T20:53:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=A 2024-11-11T20:53:24,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:24,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=B 2024-11-11T20:53:24,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:24,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=C 2024-11-11T20:53:24,842 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117915bc0dbe2e49c197e19bd21f4b1b97_759cf260483d400f5615c66d4fd85216 is 50, key is test_row_0/A:col10/1731358403674/Put/seqid=0 2024-11-11T20:53:24,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741976_1152 (size=17534) 2024-11-11T20:53:24,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,912 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,917 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117915bc0dbe2e49c197e19bd21f4b1b97_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117915bc0dbe2e49c197e19bd21f4b1b97_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:24,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:24,918 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/3d1e86127fc04f4e95d3417ec7148013, store: [table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:24,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/3d1e86127fc04f4e95d3417ec7148013 is 175, key is test_row_0/A:col10/1731358403674/Put/seqid=0 2024-11-11T20:53:24,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358404919, max timestamp=1731358404927Num uniques 859; 2024-11-11T20:53:24,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:53:24,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358404919, max timestamp=1731358404928Num uniques 859; 2024-11-11T20:53:24,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:53:24,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358404919, max timestamp=1731358404928Num uniques 859; 2024-11-11T20:53:24,931 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:53:24,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741977_1153 (size=48639) 2024-11-11T20:53:24,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358464987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:24,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:24,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358464988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:24,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:24,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:24,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:24,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:24,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:24,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:25,004 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:25,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358464999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:25,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:25,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:25,005 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:25,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:25,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358465106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:25,107 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:25,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:25,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:25,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358465106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:25,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:25,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:25,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:25,107 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:25,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:25,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358465115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:25,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:25,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:25,124 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:25,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:25,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358465316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:25,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:25,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:25,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:25,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:25,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358465317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:25,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:25,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:25,321 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:25,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:25,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358465336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:25,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:25,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:25,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:25,375 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=324, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/3d1e86127fc04f4e95d3417ec7148013 2024-11-11T20:53:25,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/fbde519ed52c47a0bf68798c6ad2f09e is 50, key is test_row_0/B:col10/1731358403674/Put/seqid=0 2024-11-11T20:53:25,410 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741978_1154 (size=12301) 2024-11-11T20:53:25,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:25,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358465626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:25,631 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:25,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358465626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:25,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:25,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:25,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:25,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:25,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:25,633 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:25,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:25,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358465656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:25,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:25,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:25,664 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:25,811 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/fbde519ed52c47a0bf68798c6ad2f09e 2024-11-11T20:53:25,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/651958deb3864f70abda4bddc5675968 is 50, key is test_row_0/C:col10/1731358403674/Put/seqid=0 2024-11-11T20:53:25,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741979_1155 (size=12301) 2024-11-11T20:53:26,136 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358466136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:26,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:26,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:26,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:26,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358466147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:26,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:26,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:26,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:26,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358466175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:26,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:26,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:26,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:26,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/651958deb3864f70abda4bddc5675968 2024-11-11T20:53:26,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/3d1e86127fc04f4e95d3417ec7148013 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/3d1e86127fc04f4e95d3417ec7148013 2024-11-11T20:53:26,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/3d1e86127fc04f4e95d3417ec7148013, entries=250, sequenceid=324, filesize=47.5 K 2024-11-11T20:53:26,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/fbde519ed52c47a0bf68798c6ad2f09e as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/fbde519ed52c47a0bf68798c6ad2f09e 2024-11-11T20:53:26,293 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/fbde519ed52c47a0bf68798c6ad2f09e, entries=150, sequenceid=324, filesize=12.0 K 2024-11-11T20:53:26,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/651958deb3864f70abda4bddc5675968 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/651958deb3864f70abda4bddc5675968 2024-11-11T20:53:26,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/651958deb3864f70abda4bddc5675968, entries=150, sequenceid=324, filesize=12.0 K 2024-11-11T20:53:26,300 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 759cf260483d400f5615c66d4fd85216 in 1459ms, sequenceid=324, compaction requested=true 2024-11-11T20:53:26,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:26,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 759cf260483d400f5615c66d4fd85216:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:53:26,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:26,300 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:26,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 759cf260483d400f5615c66d4fd85216:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:53:26,300 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:26,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:26,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 759cf260483d400f5615c66d4fd85216:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:53:26,300 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:26,301 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:26,302 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 759cf260483d400f5615c66d4fd85216/B is initiating minor compaction (all files) 2024-11-11T20:53:26,302 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 759cf260483d400f5615c66d4fd85216/B in TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:26,302 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/e646d233d4ef44d7abcb03b76eaab84a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/e16338392d45432b853e94df2a3cf995, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/fbde519ed52c47a0bf68798c6ad2f09e] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp, totalSize=36.2 K 2024-11-11T20:53:26,302 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111273 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:26,302 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 759cf260483d400f5615c66d4fd85216/A is initiating minor compaction (all files) 2024-11-11T20:53:26,302 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 759cf260483d400f5615c66d4fd85216/A in TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:26,302 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/36926a235b224be3b0223482f1d86b65, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/03e6fcd7cf864f128c15c541b4e393f9, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/3d1e86127fc04f4e95d3417ec7148013] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp, totalSize=108.7 K 2024-11-11T20:53:26,302 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:26,302 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. files: [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/36926a235b224be3b0223482f1d86b65, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/03e6fcd7cf864f128c15c541b4e393f9, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/3d1e86127fc04f4e95d3417ec7148013] 2024-11-11T20:53:26,303 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting e646d233d4ef44d7abcb03b76eaab84a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731358401931 2024-11-11T20:53:26,303 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36926a235b224be3b0223482f1d86b65, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731358401931 2024-11-11T20:53:26,304 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting e16338392d45432b853e94df2a3cf995, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731358402503 2024-11-11T20:53:26,305 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03e6fcd7cf864f128c15c541b4e393f9, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731358402503 2024-11-11T20:53:26,306 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d1e86127fc04f4e95d3417ec7148013, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731358403652 2024-11-11T20:53:26,306 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting fbde519ed52c47a0bf68798c6ad2f09e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731358403674 2024-11-11T20:53:26,328 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 759cf260483d400f5615c66d4fd85216#B#compaction#129 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:26,329 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/963b7f2537c2451c9cb3ddd153628587 is 50, key is test_row_0/B:col10/1731358403674/Put/seqid=0 2024-11-11T20:53:26,342 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:26,369 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411111621740e681144088c0836cc1880d430_759cf260483d400f5615c66d4fd85216 store=[table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:26,371 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411111621740e681144088c0836cc1880d430_759cf260483d400f5615c66d4fd85216, store=[table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:26,372 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111621740e681144088c0836cc1880d430_759cf260483d400f5615c66d4fd85216 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:26,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741980_1156 (size=12677) 2024-11-11T20:53:26,397 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/963b7f2537c2451c9cb3ddd153628587 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/963b7f2537c2451c9cb3ddd153628587 2024-11-11T20:53:26,407 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 759cf260483d400f5615c66d4fd85216/B of 759cf260483d400f5615c66d4fd85216 into 963b7f2537c2451c9cb3ddd153628587(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:26,408 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:26,408 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., storeName=759cf260483d400f5615c66d4fd85216/B, priority=13, startTime=1731358406300; duration=0sec 2024-11-11T20:53:26,409 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:26,409 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 759cf260483d400f5615c66d4fd85216:B 2024-11-11T20:53:26,409 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:26,410 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:26,410 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 759cf260483d400f5615c66d4fd85216/C is initiating minor compaction (all files) 2024-11-11T20:53:26,411 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 759cf260483d400f5615c66d4fd85216/C in TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:26,411 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/7125a843d7554e5ab1c3e472ae1a3589, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/9fc8a1dacfe04b038274bb52be949147, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/651958deb3864f70abda4bddc5675968] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp, totalSize=36.2 K 2024-11-11T20:53:26,411 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 7125a843d7554e5ab1c3e472ae1a3589, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=240, earliestPutTs=1731358401931 2024-11-11T20:53:26,412 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fc8a1dacfe04b038274bb52be949147, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731358402503 2024-11-11T20:53:26,412 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 651958deb3864f70abda4bddc5675968, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731358403674 2024-11-11T20:53:26,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741981_1157 (size=4469) 2024-11-11T20:53:26,448 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 759cf260483d400f5615c66d4fd85216#C#compaction#131 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:26,448 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/4ed07674d4ff444da4057fc0e36478e9 is 50, key is test_row_0/C:col10/1731358403674/Put/seqid=0 2024-11-11T20:53:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-11T20:53:26,466 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:53:26,467 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:53:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-11T20:53:26,469 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:53:26,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T20:53:26,473 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:53:26,473 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:53:26,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741982_1158 (size=12677) 2024-11-11T20:53:26,488 INFO [master/51ca66f7ee3c:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-11T20:53:26,488 INFO [master/51ca66f7ee3c:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-11T20:53:26,488 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/4ed07674d4ff444da4057fc0e36478e9 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/4ed07674d4ff444da4057fc0e36478e9 2024-11-11T20:53:26,500 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 759cf260483d400f5615c66d4fd85216/C of 759cf260483d400f5615c66d4fd85216 into 4ed07674d4ff444da4057fc0e36478e9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:26,500 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:26,500 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., storeName=759cf260483d400f5615c66d4fd85216/C, priority=13, startTime=1731358406300; duration=0sec 2024-11-11T20:53:26,500 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:26,500 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 759cf260483d400f5615c66d4fd85216:C 2024-11-11T20:53:26,553 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 759cf260483d400f5615c66d4fd85216 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-11T20:53:26,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=A 2024-11-11T20:53:26,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T20:53:26,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=B 2024-11-11T20:53:26,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T20:53:26,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=C 2024-11-11T20:53:26,553 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T20:53:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T20:53:26,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:26,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358466570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:26,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:26,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:26,578 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:26,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:26,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358466578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:26,582 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:26,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111192ef6e7e8f18472b95bbac85043a5db6_759cf260483d400f5615c66d4fd85216 is 50, key is test_row_0/A:col10/1731358406550/Put/seqid=0 2024-11-11T20:53:26,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T20:53:26,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:26,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:26,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:26,628 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:26,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:26,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741983_1159 (size=12454) 2024-11-11T20:53:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:26,630 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:26,635 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111192ef6e7e8f18472b95bbac85043a5db6_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111192ef6e7e8f18472b95bbac85043a5db6_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:26,636 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/db71af42ae3e4d949df364e19aee952a, store: [table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:26,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/db71af42ae3e4d949df364e19aee952a is 175, key is test_row_0/A:col10/1731358406550/Put/seqid=0 2024-11-11T20:53:26,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741984_1160 (size=31255) 2024-11-11T20:53:26,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358466687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:26,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:26,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358466687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:26,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:26,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:26,688 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:26,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:26,688 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:26,689 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:26,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T20:53:26,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:26,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:26,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:26,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:26,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:26,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:26,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T20:53:26,827 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 759cf260483d400f5615c66d4fd85216#A#compaction#130 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:26,827 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/eca246cada6845ffbe2986bfd8c007d4 is 175, key is test_row_0/A:col10/1731358403674/Put/seqid=0 2024-11-11T20:53:26,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741985_1161 (size=31631) 2024-11-11T20:53:26,842 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/eca246cada6845ffbe2986bfd8c007d4 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/eca246cada6845ffbe2986bfd8c007d4 2024-11-11T20:53:26,848 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 759cf260483d400f5615c66d4fd85216/A of 759cf260483d400f5615c66d4fd85216 into eca246cada6845ffbe2986bfd8c007d4(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:26,848 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:26,848 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., storeName=759cf260483d400f5615c66d4fd85216/A, priority=13, startTime=1731358406300; duration=0sec 2024-11-11T20:53:26,848 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:26,848 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 759cf260483d400f5615c66d4fd85216:A 2024-11-11T20:53:26,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:26,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358466897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:26,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:26,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358466896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:26,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:26,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:26,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:26,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:26,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:26,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:26,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T20:53:26,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:26,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:26,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:26,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:26,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:26,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:27,072 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=364, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/db71af42ae3e4d949df364e19aee952a 2024-11-11T20:53:27,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/5122080a859c4167a6611507111d6d0f is 50, key is test_row_0/B:col10/1731358406550/Put/seqid=0 2024-11-11T20:53:27,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T20:53:27,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:27,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:27,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:27,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:27,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:27,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T20:53:27,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741986_1162 (size=12301) 2024-11-11T20:53:27,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358467146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:27,149 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_0' in 759cf260483d400f5615c66d4fd85216 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:27,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:27,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:27,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:27,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:27,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358467165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:27,168 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_0' in 759cf260483d400f5615c66d4fd85216 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:27,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:27,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:27,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358467186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:27,189 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_1' in 759cf260483d400f5615c66d4fd85216 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:27,190 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:27,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358467206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:27,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:27,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358467206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:27,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:27,208 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:27,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:27,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:27,208 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:27,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T20:53:27,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:27,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:27,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:27,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:27,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:27,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:27,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T20:53:27,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:27,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:27,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:27,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:27,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:27,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:27,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/5122080a859c4167a6611507111d6d0f 2024-11-11T20:53:27,535 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/d91164286e6f4bf6947c8a452a635260 is 50, key is test_row_0/C:col10/1731358406550/Put/seqid=0 2024-11-11T20:53:27,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T20:53:27,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:27,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:27,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:27,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:27,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:27,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:27,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741987_1163 (size=12301) 2024-11-11T20:53:27,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T20:53:27,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T20:53:27,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:27,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:27,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:27,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:27,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:27,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:27,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:27,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358467716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:27,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:27,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358467716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:27,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:27,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:27,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:27,718 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:27,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T20:53:27,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:27,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:27,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:27,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:27,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:27,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:27,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/d91164286e6f4bf6947c8a452a635260 2024-11-11T20:53:27,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/db71af42ae3e4d949df364e19aee952a as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/db71af42ae3e4d949df364e19aee952a 2024-11-11T20:53:27,982 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/db71af42ae3e4d949df364e19aee952a, entries=150, sequenceid=364, filesize=30.5 K 2024-11-11T20:53:27,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/5122080a859c4167a6611507111d6d0f as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/5122080a859c4167a6611507111d6d0f 2024-11-11T20:53:27,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/5122080a859c4167a6611507111d6d0f, entries=150, sequenceid=364, filesize=12.0 K 2024-11-11T20:53:27,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/d91164286e6f4bf6947c8a452a635260 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/d91164286e6f4bf6947c8a452a635260 2024-11-11T20:53:28,005 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/d91164286e6f4bf6947c8a452a635260, entries=150, sequenceid=364, filesize=12.0 K 2024-11-11T20:53:28,006 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 759cf260483d400f5615c66d4fd85216 in 1453ms, sequenceid=364, compaction requested=false 2024-11-11T20:53:28,006 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:28,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-11T20:53:28,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:28,011 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 759cf260483d400f5615c66d4fd85216 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-11T20:53:28,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=A 2024-11-11T20:53:28,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:28,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=B 2024-11-11T20:53:28,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:28,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=C 2024-11-11T20:53:28,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:28,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111266cabcf643243bebd090add36423d35_759cf260483d400f5615c66d4fd85216 is 50, key is test_row_0/A:col10/1731358406557/Put/seqid=0 2024-11-11T20:53:28,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741988_1164 (size=12454) 2024-11-11T20:53:28,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:28,468 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111266cabcf643243bebd090add36423d35_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111266cabcf643243bebd090add36423d35_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:28,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/040aede3841247ec97a2ae5f9a2bdc1a, store: [table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:28,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/040aede3841247ec97a2ae5f9a2bdc1a is 175, key is test_row_0/A:col10/1731358406557/Put/seqid=0 2024-11-11T20:53:28,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741989_1165 (size=31255) 2024-11-11T20:53:28,500 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=374, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/040aede3841247ec97a2ae5f9a2bdc1a 2024-11-11T20:53:28,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/33d248ba49ff4f01ad743510b55744b2 is 50, key is test_row_0/B:col10/1731358406557/Put/seqid=0 2024-11-11T20:53:28,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741990_1166 (size=12301) 2024-11-11T20:53:28,525 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/33d248ba49ff4f01ad743510b55744b2 2024-11-11T20:53:28,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/07e0cdce823c4d38bf302b9aaec8d836 is 50, key is test_row_0/C:col10/1731358406557/Put/seqid=0 2024-11-11T20:53:28,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741991_1167 (size=12301) 2024-11-11T20:53:28,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T20:53:28,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:28,789 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358408759, max timestamp=1731358408786Num uniques 859; 2024-11-11T20:53:28,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:53:28,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358408759, max timestamp=1731358408786Num uniques 859; 2024-11-11T20:53:28,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:53:28,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358408759, max timestamp=1731358408786Num uniques 859; 2024-11-11T20:53:28,790 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:53:28,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:28,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358468846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:28,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:28,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:28,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:28,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:28,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358468855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:28,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:28,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:28,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:28,957 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/07e0cdce823c4d38bf302b9aaec8d836 2024-11-11T20:53:28,961 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/040aede3841247ec97a2ae5f9a2bdc1a as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/040aede3841247ec97a2ae5f9a2bdc1a 2024-11-11T20:53:28,967 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/040aede3841247ec97a2ae5f9a2bdc1a, entries=150, sequenceid=374, filesize=30.5 K 2024-11-11T20:53:28,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/33d248ba49ff4f01ad743510b55744b2 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/33d248ba49ff4f01ad743510b55744b2 2024-11-11T20:53:28,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:28,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358468966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:28,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:28,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358468966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:28,970 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:28,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:28,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:28,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:28,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:28,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:28,975 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/33d248ba49ff4f01ad743510b55744b2, entries=150, sequenceid=374, filesize=12.0 K 2024-11-11T20:53:28,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/07e0cdce823c4d38bf302b9aaec8d836 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/07e0cdce823c4d38bf302b9aaec8d836 2024-11-11T20:53:28,982 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/07e0cdce823c4d38bf302b9aaec8d836, entries=150, sequenceid=374, filesize=12.0 K 2024-11-11T20:53:28,983 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 759cf260483d400f5615c66d4fd85216 in 973ms, sequenceid=374, compaction requested=true 2024-11-11T20:53:28,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:28,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:28,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-11T20:53:28,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-11T20:53:28,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-11T20:53:28,987 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5120 sec 2024-11-11T20:53:28,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 2.5210 sec 2024-11-11T20:53:29,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:29,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 759cf260483d400f5615c66d4fd85216 3/3 column families, dataSize=154.31 KB heapSize=440.77 KB 2024-11-11T20:53:29,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=A 2024-11-11T20:53:29,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T20:53:29,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=B 2024-11-11T20:53:29,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T20:53:29,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=C 2024-11-11T20:53:29,179 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T20:53:29,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:29,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358469191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:29,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,195 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:29,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358469192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:29,199 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:29,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358469194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:29,199 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,200 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:29,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117e14dd1cc85b4631a7dbf33a33d53cd4_759cf260483d400f5615c66d4fd85216 is 50, key is test_row_0/A:col10/1731358409177/Put/seqid=0 2024-11-11T20:53:29,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:29,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:29,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358469196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:29,203 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:29,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358469197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:29,203 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_1' in 759cf260483d400f5615c66d4fd85216 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,204 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:29,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741992_1168 (size=17534) 2024-11-11T20:53:29,221 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,229 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117e14dd1cc85b4631a7dbf33a33d53cd4_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117e14dd1cc85b4631a7dbf33a33d53cd4_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:29,231 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/d3f656392be5477089aa91d243da823c, store: [table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:29,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/d3f656392be5477089aa91d243da823c is 175, key is test_row_0/A:col10/1731358409177/Put/seqid=0 2024-11-11T20:53:29,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741993_1169 (size=48639) 2024-11-11T20:53:29,261 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=413, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/d3f656392be5477089aa91d243da823c 2024-11-11T20:53:29,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/29d8d6f78c7c401f8ec7b9c4a6282a19 is 50, key is test_row_0/B:col10/1731358409177/Put/seqid=0 2024-11-11T20:53:29,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741994_1170 (size=12301) 2024-11-11T20:53:29,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358469306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:29,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358469306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:29,311 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:29,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:29,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358469306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:29,312 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:29,312 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:29,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358469306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:29,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,312 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:29,516 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:29,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358469516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:29,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,517 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:29,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:29,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358469516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:29,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:29,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358469516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:29,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,519 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:29,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:29,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358469526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:29,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,533 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:29,684 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/29d8d6f78c7c401f8ec7b9c4a6282a19 2024-11-11T20:53:29,698 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/9633ec3769594686a19a974599e23545 is 50, key is test_row_0/C:col10/1731358409177/Put/seqid=0 2024-11-11T20:53:29,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741995_1171 (size=12301) 2024-11-11T20:53:29,710 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/9633ec3769594686a19a974599e23545 2024-11-11T20:53:29,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/d3f656392be5477089aa91d243da823c as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/d3f656392be5477089aa91d243da823c 2024-11-11T20:53:29,729 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/d3f656392be5477089aa91d243da823c, entries=250, sequenceid=413, filesize=47.5 K 2024-11-11T20:53:29,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/29d8d6f78c7c401f8ec7b9c4a6282a19 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/29d8d6f78c7c401f8ec7b9c4a6282a19 2024-11-11T20:53:29,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/29d8d6f78c7c401f8ec7b9c4a6282a19, entries=150, sequenceid=413, filesize=12.0 K 2024-11-11T20:53:29,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,739 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/9633ec3769594686a19a974599e23545 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/9633ec3769594686a19a974599e23545 2024-11-11T20:53:29,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/9633ec3769594686a19a974599e23545, entries=150, sequenceid=413, filesize=12.0 K 2024-11-11T20:53:29,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,749 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~440.72 KB/451296, currentSize=33.54 KB/34350 for 759cf260483d400f5615c66d4fd85216 in 571ms, sequenceid=413, compaction requested=true 2024-11-11T20:53:29,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:29,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 759cf260483d400f5615c66d4fd85216:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:53:29,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:29,750 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:53:29,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 759cf260483d400f5615c66d4fd85216:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:53:29,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:29,750 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:53:29,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 759cf260483d400f5615c66d4fd85216:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:53:29,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:29,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,751 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142780 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:53:29,752 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 759cf260483d400f5615c66d4fd85216/A is initiating minor compaction (all files) 2024-11-11T20:53:29,752 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 759cf260483d400f5615c66d4fd85216/A in TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:29,752 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49580 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:53:29,752 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 759cf260483d400f5615c66d4fd85216/B is initiating minor compaction (all files) 2024-11-11T20:53:29,752 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/eca246cada6845ffbe2986bfd8c007d4, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/db71af42ae3e4d949df364e19aee952a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/040aede3841247ec97a2ae5f9a2bdc1a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/d3f656392be5477089aa91d243da823c] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp, totalSize=139.4 K 2024-11-11T20:53:29,752 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 759cf260483d400f5615c66d4fd85216/B in TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:29,752 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:29,752 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/963b7f2537c2451c9cb3ddd153628587, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/5122080a859c4167a6611507111d6d0f, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/33d248ba49ff4f01ad743510b55744b2, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/29d8d6f78c7c401f8ec7b9c4a6282a19] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp, totalSize=48.4 K 2024-11-11T20:53:29,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,752 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. files: [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/eca246cada6845ffbe2986bfd8c007d4, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/db71af42ae3e4d949df364e19aee952a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/040aede3841247ec97a2ae5f9a2bdc1a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/d3f656392be5477089aa91d243da823c] 2024-11-11T20:53:29,753 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 963b7f2537c2451c9cb3ddd153628587, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731358403674 2024-11-11T20:53:29,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,753 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting eca246cada6845ffbe2986bfd8c007d4, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731358403674 2024-11-11T20:53:29,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,754 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 5122080a859c4167a6611507111d6d0f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731358404982 2024-11-11T20:53:29,754 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting db71af42ae3e4d949df364e19aee952a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731358404982 2024-11-11T20:53:29,754 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 33d248ba49ff4f01ad743510b55744b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731358406557 2024-11-11T20:53:29,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,755 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 29d8d6f78c7c401f8ec7b9c4a6282a19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731358408852 2024-11-11T20:53:29,755 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 040aede3841247ec97a2ae5f9a2bdc1a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731358406557 2024-11-11T20:53:29,755 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3f656392be5477089aa91d243da823c, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731358408827 2024-11-11T20:53:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,774 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:29,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,787 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 759cf260483d400f5615c66d4fd85216#B#compaction#142 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,788 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/1cb7a5ba45aa49c3946f5b0a17ef84f1 is 50, key is test_row_0/B:col10/1731358409177/Put/seqid=0 2024-11-11T20:53:29,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,789 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411113f63a41622704353b67cfc242f09e47b_759cf260483d400f5615c66d4fd85216 store=[table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:29,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,798 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411113f63a41622704353b67cfc242f09e47b_759cf260483d400f5615c66d4fd85216, store=[table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:29,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,798 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113f63a41622704353b67cfc242f09e47b_759cf260483d400f5615c66d4fd85216 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:29,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741996_1172 (size=12813) 2024-11-11T20:53:29,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,848 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/1cb7a5ba45aa49c3946f5b0a17ef84f1 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/1cb7a5ba45aa49c3946f5b0a17ef84f1 2024-11-11T20:53:29,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741997_1173 (size=4469) 2024-11-11T20:53:29,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,851 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 759cf260483d400f5615c66d4fd85216#A#compaction#141 average throughput is 0.32 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:29,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,852 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/6b0dfa57297a4132b492635b7a1d4fcf is 175, key is test_row_0/A:col10/1731358409177/Put/seqid=0 2024-11-11T20:53:29,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:29,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 759cf260483d400f5615c66d4fd85216 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T20:53:29,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=A 2024-11-11T20:53:29,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:29,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=B 2024-11-11T20:53:29,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:29,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=C 2024-11-11T20:53:29,854 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:29,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,861 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 759cf260483d400f5615c66d4fd85216/B of 759cf260483d400f5615c66d4fd85216 into 1cb7a5ba45aa49c3946f5b0a17ef84f1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:29,862 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:29,862 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., storeName=759cf260483d400f5615c66d4fd85216/B, priority=12, startTime=1731358409750; duration=0sec 2024-11-11T20:53:29,862 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:29,862 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 759cf260483d400f5615c66d4fd85216:B 2024-11-11T20:53:29,862 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:53:29,864 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49580 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:53:29,864 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 759cf260483d400f5615c66d4fd85216/C is initiating minor compaction (all files) 2024-11-11T20:53:29,864 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 759cf260483d400f5615c66d4fd85216/C in TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:29,864 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/4ed07674d4ff444da4057fc0e36478e9, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/d91164286e6f4bf6947c8a452a635260, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/07e0cdce823c4d38bf302b9aaec8d836, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/9633ec3769594686a19a974599e23545] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp, totalSize=48.4 K 2024-11-11T20:53:29,865 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ed07674d4ff444da4057fc0e36478e9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731358403674 2024-11-11T20:53:29,865 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting d91164286e6f4bf6947c8a452a635260, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731358404982 2024-11-11T20:53:29,866 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 07e0cdce823c4d38bf302b9aaec8d836, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1731358406557 2024-11-11T20:53:29,867 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 9633ec3769594686a19a974599e23545, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731358408852 2024-11-11T20:53:29,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741998_1174 (size=31767) 2024-11-11T20:53:29,913 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 759cf260483d400f5615c66d4fd85216#C#compaction#143 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:29,914 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/2c5733a553604bb4b009f25de607c90b is 50, key is test_row_0/C:col10/1731358409177/Put/seqid=0 2024-11-11T20:53:29,915 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/6b0dfa57297a4132b492635b7a1d4fcf as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/6b0dfa57297a4132b492635b7a1d4fcf 2024-11-11T20:53:29,916 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b20121690c9546e4b97efa339c1a871a_759cf260483d400f5615c66d4fd85216 is 50, key is test_row_0/A:col10/1731358409193/Put/seqid=0 2024-11-11T20:53:29,925 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 759cf260483d400f5615c66d4fd85216/A of 759cf260483d400f5615c66d4fd85216 into 6b0dfa57297a4132b492635b7a1d4fcf(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:29,925 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:29,925 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., storeName=759cf260483d400f5615c66d4fd85216/A, priority=12, startTime=1731358409750; duration=0sec 2024-11-11T20:53:29,925 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:29,925 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 759cf260483d400f5615c66d4fd85216:A 2024-11-11T20:53:29,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731358409899, max timestamp=1731358409934Num uniques 859; 2024-11-11T20:53:29,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T20:53:29,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731358409899, max timestamp=1731358409934Num uniques 859; 2024-11-11T20:53:29,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T20:53:29,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731358409899, max timestamp=1731358409934Num uniques 859; 2024-11-11T20:53:29,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T20:53:29,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741999_1175 (size=12813) 2024-11-11T20:53:29,965 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/2c5733a553604bb4b009f25de607c90b as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/2c5733a553604bb4b009f25de607c90b 2024-11-11T20:53:29,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:29,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358469961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:29,966 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:29,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,967 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:29,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358469964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:29,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:29,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,967 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:29,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358469964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:29,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,967 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,968 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:29,973 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 759cf260483d400f5615c66d4fd85216/C of 759cf260483d400f5615c66d4fd85216 into 2c5733a553604bb4b009f25de607c90b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:29,973 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:29,973 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:29,973 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., storeName=759cf260483d400f5615c66d4fd85216/C, priority=12, startTime=1731358409750; duration=0sec 2024-11-11T20:53:29,973 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:29,973 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 759cf260483d400f5615c66d4fd85216:C 2024-11-11T20:53:29,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358469966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:29,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:29,974 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:29,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742000_1176 (size=14994) 2024-11-11T20:53:29,977 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:29,982 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b20121690c9546e4b97efa339c1a871a_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b20121690c9546e4b97efa339c1a871a_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:29,983 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/cc319fb3c1c74396bb651bdd64874ed2, store: [table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:29,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/cc319fb3c1c74396bb651bdd64874ed2 is 175, key is test_row_0/A:col10/1731358409193/Put/seqid=0 2024-11-11T20:53:30,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742001_1177 (size=39949) 2024-11-11T20:53:30,023 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=423, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/cc319fb3c1c74396bb651bdd64874ed2 2024-11-11T20:53:30,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/b2b4bb68aae5427a845ce05ac255e23a is 50, key is test_row_0/B:col10/1731358409193/Put/seqid=0 2024-11-11T20:53:30,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742002_1178 (size=12301) 2024-11-11T20:53:30,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/b2b4bb68aae5427a845ce05ac255e23a 2024-11-11T20:53:30,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:30,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358470075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:30,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,077 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:30,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:30,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358470076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:30,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:30,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,081 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:30,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358470076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:30,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,081 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:30,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:30,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358470086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:30,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,089 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:30,090 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/9dadf913473e4bb49911755e3b1972dc is 50, key is test_row_0/C:col10/1731358409193/Put/seqid=0 2024-11-11T20:53:30,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742003_1179 (size=12301) 2024-11-11T20:53:30,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/9dadf913473e4bb49911755e3b1972dc 2024-11-11T20:53:30,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/cc319fb3c1c74396bb651bdd64874ed2 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/cc319fb3c1c74396bb651bdd64874ed2 2024-11-11T20:53:30,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/cc319fb3c1c74396bb651bdd64874ed2, entries=200, sequenceid=423, filesize=39.0 K 2024-11-11T20:53:30,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/b2b4bb68aae5427a845ce05ac255e23a as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/b2b4bb68aae5427a845ce05ac255e23a 2024-11-11T20:53:30,141 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/b2b4bb68aae5427a845ce05ac255e23a, entries=150, sequenceid=423, filesize=12.0 K 2024-11-11T20:53:30,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/9dadf913473e4bb49911755e3b1972dc as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/9dadf913473e4bb49911755e3b1972dc 2024-11-11T20:53:30,155 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/9dadf913473e4bb49911755e3b1972dc, entries=150, sequenceid=423, filesize=12.0 K 2024-11-11T20:53:30,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for 759cf260483d400f5615c66d4fd85216 in 302ms, sequenceid=423, compaction requested=false 2024-11-11T20:53:30,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:30,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 759cf260483d400f5615c66d4fd85216 3/3 column families, dataSize=154.31 KB heapSize=426 KB 2024-11-11T20:53:30,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=A 2024-11-11T20:53:30,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:30,288 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:30,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=B 2024-11-11T20:53:30,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:30,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=C 2024-11-11T20:53:30,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:30,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111164b36427dd794a3da851c9294524fe70_759cf260483d400f5615c66d4fd85216 is 50, key is test_row_0/A:col10/1731358410286/Put/seqid=0 2024-11-11T20:53:30,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:30,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358470298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:30,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,304 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:30,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:30,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358470299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:30,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:30,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358470301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:30,305 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:30,305 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:30,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358470302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:30,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,306 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:30,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:30,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742004_1180 (size=17534) 2024-11-11T20:53:30,314 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:30,319 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111164b36427dd794a3da851c9294524fe70_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111164b36427dd794a3da851c9294524fe70_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:30,324 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/315c8bfd672747bdbf6e54112a4a5fd0, store: [table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:30,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/315c8bfd672747bdbf6e54112a4a5fd0 is 175, key is test_row_0/A:col10/1731358410286/Put/seqid=0 2024-11-11T20:53:30,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742005_1181 (size=48639) 2024-11-11T20:53:30,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:30,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358470406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:30,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,410 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:30,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:30,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358470415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:30,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,418 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:30,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:30,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358470415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:30,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:30,418 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:30,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358470415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,419 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:30,621 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:30,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358470617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:30,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:30,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358470626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:30,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:30,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358470626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:30,627 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,628 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:30,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:30,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358470626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:30,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,628 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:30,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,628 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:30,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-11T20:53:30,636 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:53:30,637 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:53:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-11T20:53:30,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T20:53:30,638 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:53:30,639 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:53:30,639 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:53:30,734 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=462, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/315c8bfd672747bdbf6e54112a4a5fd0 2024-11-11T20:53:30,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/bff4dd52e31e40a5bcd3bb792e079ff3 is 50, key is test_row_0/B:col10/1731358410286/Put/seqid=0 2024-11-11T20:53:30,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T20:53:30,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742006_1182 (size=12301) 2024-11-11T20:53:30,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T20:53:30,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:30,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:30,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:30,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:30,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:30,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:30,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:30,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358470925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:30,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:30,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:30,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358470935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:30,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:30,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,937 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:30,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358470936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:30,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:30,937 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:30,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358470936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:30,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:30,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:30,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T20:53:30,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:30,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:30,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:30,944 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:30,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:30,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:30,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T20:53:31,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T20:53:31,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:31,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:31,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:31,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:31,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:31,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:31,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/bff4dd52e31e40a5bcd3bb792e079ff3 2024-11-11T20:53:31,165 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/851253dc500e49dcb9de3259e3e35b7c is 50, key is test_row_0/C:col10/1731358410286/Put/seqid=0 2024-11-11T20:53:31,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742007_1183 (size=12301) 2024-11-11T20:53:31,191 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=462 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/851253dc500e49dcb9de3259e3e35b7c 2024-11-11T20:53:31,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/315c8bfd672747bdbf6e54112a4a5fd0 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/315c8bfd672747bdbf6e54112a4a5fd0 2024-11-11T20:53:31,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/315c8bfd672747bdbf6e54112a4a5fd0, entries=250, sequenceid=462, filesize=47.5 K 2024-11-11T20:53:31,234 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/bff4dd52e31e40a5bcd3bb792e079ff3 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/bff4dd52e31e40a5bcd3bb792e079ff3 2024-11-11T20:53:31,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/bff4dd52e31e40a5bcd3bb792e079ff3, entries=150, sequenceid=462, filesize=12.0 K 2024-11-11T20:53:31,242 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/851253dc500e49dcb9de3259e3e35b7c as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/851253dc500e49dcb9de3259e3e35b7c 2024-11-11T20:53:31,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T20:53:31,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:31,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:31,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:31,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:31,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:31,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:31,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/851253dc500e49dcb9de3259e3e35b7c, entries=150, sequenceid=462, filesize=12.0 K 2024-11-11T20:53:31,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~425.95 KB/436176, currentSize=33.54 KB/34350 for 759cf260483d400f5615c66d4fd85216 in 968ms, sequenceid=462, compaction requested=true 2024-11-11T20:53:31,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:31,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 759cf260483d400f5615c66d4fd85216:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:53:31,258 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:31,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:31,258 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:31,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 759cf260483d400f5615c66d4fd85216:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:53:31,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:31,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 759cf260483d400f5615c66d4fd85216:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:53:31,258 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:31,260 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:31,260 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 759cf260483d400f5615c66d4fd85216/B is initiating minor compaction (all files) 2024-11-11T20:53:31,260 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 759cf260483d400f5615c66d4fd85216/B in TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:31,260 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/1cb7a5ba45aa49c3946f5b0a17ef84f1, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/b2b4bb68aae5427a845ce05ac255e23a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/bff4dd52e31e40a5bcd3bb792e079ff3] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp, totalSize=36.5 K 2024-11-11T20:53:31,260 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120355 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:31,260 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 759cf260483d400f5615c66d4fd85216/A is initiating minor compaction (all files) 2024-11-11T20:53:31,261 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 759cf260483d400f5615c66d4fd85216/A in TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:31,261 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/6b0dfa57297a4132b492635b7a1d4fcf, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/cc319fb3c1c74396bb651bdd64874ed2, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/315c8bfd672747bdbf6e54112a4a5fd0] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp, totalSize=117.5 K 2024-11-11T20:53:31,261 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:31,261 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. files: [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/6b0dfa57297a4132b492635b7a1d4fcf, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/cc319fb3c1c74396bb651bdd64874ed2, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/315c8bfd672747bdbf6e54112a4a5fd0] 2024-11-11T20:53:31,263 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cb7a5ba45aa49c3946f5b0a17ef84f1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731358408852 2024-11-11T20:53:31,264 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting b2b4bb68aae5427a845ce05ac255e23a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731358409193 2024-11-11T20:53:31,264 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b0dfa57297a4132b492635b7a1d4fcf, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731358408852 2024-11-11T20:53:31,265 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting bff4dd52e31e40a5bcd3bb792e079ff3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731358409962 2024-11-11T20:53:31,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T20:53:31,268 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc319fb3c1c74396bb651bdd64874ed2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731358409190 2024-11-11T20:53:31,269 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 315c8bfd672747bdbf6e54112a4a5fd0, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731358409960 2024-11-11T20:53:31,291 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 759cf260483d400f5615c66d4fd85216#B#compaction#150 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:31,292 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/bcb75ea0c1f1409fad465fb9efdaff8b is 50, key is test_row_0/B:col10/1731358410286/Put/seqid=0 2024-11-11T20:53:31,294 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:31,304 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111ff7fc97ffd4447e68dc2f12a624b7050_759cf260483d400f5615c66d4fd85216 store=[table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:31,307 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111ff7fc97ffd4447e68dc2f12a624b7050_759cf260483d400f5615c66d4fd85216, store=[table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:31,307 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ff7fc97ffd4447e68dc2f12a624b7050_759cf260483d400f5615c66d4fd85216 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:31,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742008_1184 (size=12915) 2024-11-11T20:53:31,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742009_1185 (size=4469) 2024-11-11T20:53:31,357 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 759cf260483d400f5615c66d4fd85216#A#compaction#151 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:31,358 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/35df194388ce48cbb174af19d154bf08 is 175, key is test_row_0/A:col10/1731358410286/Put/seqid=0 2024-11-11T20:53:31,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742010_1186 (size=31869) 2024-11-11T20:53:31,379 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/35df194388ce48cbb174af19d154bf08 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/35df194388ce48cbb174af19d154bf08 2024-11-11T20:53:31,393 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 759cf260483d400f5615c66d4fd85216/A of 759cf260483d400f5615c66d4fd85216 into 35df194388ce48cbb174af19d154bf08(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:31,394 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:31,394 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., storeName=759cf260483d400f5615c66d4fd85216/A, priority=13, startTime=1731358411258; duration=0sec 2024-11-11T20:53:31,394 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:31,394 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 759cf260483d400f5615c66d4fd85216:A 2024-11-11T20:53:31,395 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:31,396 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:31,396 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 759cf260483d400f5615c66d4fd85216/C is initiating minor compaction (all files) 2024-11-11T20:53:31,396 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 759cf260483d400f5615c66d4fd85216/C in TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:31,396 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/2c5733a553604bb4b009f25de607c90b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/9dadf913473e4bb49911755e3b1972dc, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/851253dc500e49dcb9de3259e3e35b7c] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp, totalSize=36.5 K 2024-11-11T20:53:31,396 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c5733a553604bb4b009f25de607c90b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=413, earliestPutTs=1731358408852 2024-11-11T20:53:31,397 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9dadf913473e4bb49911755e3b1972dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731358409193 2024-11-11T20:53:31,397 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 851253dc500e49dcb9de3259e3e35b7c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731358409962 2024-11-11T20:53:31,407 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 759cf260483d400f5615c66d4fd85216#C#compaction#152 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:31,407 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/97c1b4db81bd4685920cb8e15692daea is 50, key is test_row_0/C:col10/1731358410286/Put/seqid=0 2024-11-11T20:53:31,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T20:53:31,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:31,408 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 759cf260483d400f5615c66d4fd85216 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T20:53:31,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=A 2024-11-11T20:53:31,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:31,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=B 2024-11-11T20:53:31,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:31,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=C 2024-11-11T20:53:31,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:31,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742011_1187 (size=12915) 2024-11-11T20:53:31,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ae8cc16fd8664069af46e8f6bec9341a_759cf260483d400f5615c66d4fd85216 is 50, key is test_row_0/A:col10/1731358410299/Put/seqid=0 2024-11-11T20:53:31,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742012_1188 (size=12454) 2024-11-11T20:53:31,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:31,439 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:31,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1731358411440, max timestamp=1731358411465Num uniques 859; 2024-11-11T20:53:31,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-11T20:53:31,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1731358411440, max timestamp=1731358411465Num uniques 859; 2024-11-11T20:53:31,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-11T20:53:31,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=459, cellSize=21015, totalHeapSize=47976, min timestamp=1731358411440, max timestamp=1731358411465Num uniques 859; 2024-11-11T20:53:31,467 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=21015, suffix heap size=117064,new segment heap size=47976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=459 2024-11-11T20:53:31,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358471484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:31,489 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:31,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358471485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:31,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:31,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:31,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:31,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358471486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:31,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:31,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358471488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:31,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:31,596 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358471596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:31,597 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:31,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:31,597 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:31,597 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358471596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:31,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:31,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358471596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:31,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:31,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:31,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:31,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358471596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:31,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:31,746 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/bcb75ea0c1f1409fad465fb9efdaff8b as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/bcb75ea0c1f1409fad465fb9efdaff8b 2024-11-11T20:53:31,752 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 759cf260483d400f5615c66d4fd85216/B of 759cf260483d400f5615c66d4fd85216 into bcb75ea0c1f1409fad465fb9efdaff8b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:31,752 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:31,752 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., storeName=759cf260483d400f5615c66d4fd85216/B, priority=13, startTime=1731358411258; duration=0sec 2024-11-11T20:53:31,752 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:31,752 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 759cf260483d400f5615c66d4fd85216:B 2024-11-11T20:53:31,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T20:53:31,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:31,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358471806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:31,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:31,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:31,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358471806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:31,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:31,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:31,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358471806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:31,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:31,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:31,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358471806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:31,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:31,817 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/97c1b4db81bd4685920cb8e15692daea as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/97c1b4db81bd4685920cb8e15692daea 2024-11-11T20:53:31,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:31,826 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 759cf260483d400f5615c66d4fd85216/C of 759cf260483d400f5615c66d4fd85216 into 97c1b4db81bd4685920cb8e15692daea(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:31,826 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:31,826 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., storeName=759cf260483d400f5615c66d4fd85216/C, priority=13, startTime=1731358411258; duration=0sec 2024-11-11T20:53:31,826 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:31,826 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 759cf260483d400f5615c66d4fd85216:C 2024-11-11T20:53:31,826 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ae8cc16fd8664069af46e8f6bec9341a_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ae8cc16fd8664069af46e8f6bec9341a_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:31,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/fa9038b799c24a1eb0861e87f2fc9de3, store: [table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:31,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/fa9038b799c24a1eb0861e87f2fc9de3 is 175, key is test_row_0/A:col10/1731358410299/Put/seqid=0 2024-11-11T20:53:31,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742013_1189 (size=31255) 2024-11-11T20:53:32,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:32,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358472126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:32,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:32,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358472126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:32,126 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:32,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:32,127 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:32,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:32,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:32,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:32,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:32,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358472126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:32,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:32,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:32,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:32,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358472126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:32,127 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:32,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:32,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:32,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:32,239 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=471, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/fa9038b799c24a1eb0861e87f2fc9de3 2024-11-11T20:53:32,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/0653962dff64429ab9f348544216cf3a is 50, key is test_row_0/B:col10/1731358410299/Put/seqid=0 2024-11-11T20:53:32,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742014_1190 (size=12301) 2024-11-11T20:53:32,260 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/0653962dff64429ab9f348544216cf3a 2024-11-11T20:53:32,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/c9bcea4adca84e358a94cde943318653 is 50, key is test_row_0/C:col10/1731358410299/Put/seqid=0 2024-11-11T20:53:32,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742015_1191 (size=12301) 2024-11-11T20:53:32,637 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:32,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358472636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:32,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:32,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:32,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358472636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:32,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:32,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:32,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358472636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:32,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:32,638 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:32,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:32,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:32,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:32,638 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:32,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358472636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:32,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:32,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:32,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:32,682 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=471 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/c9bcea4adca84e358a94cde943318653 2024-11-11T20:53:32,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/fa9038b799c24a1eb0861e87f2fc9de3 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/fa9038b799c24a1eb0861e87f2fc9de3 2024-11-11T20:53:32,690 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/fa9038b799c24a1eb0861e87f2fc9de3, entries=150, sequenceid=471, filesize=30.5 K 2024-11-11T20:53:32,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/0653962dff64429ab9f348544216cf3a as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/0653962dff64429ab9f348544216cf3a 2024-11-11T20:53:32,697 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/0653962dff64429ab9f348544216cf3a, entries=150, sequenceid=471, filesize=12.0 K 2024-11-11T20:53:32,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/c9bcea4adca84e358a94cde943318653 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/c9bcea4adca84e358a94cde943318653 2024-11-11T20:53:32,705 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/c9bcea4adca84e358a94cde943318653, entries=150, sequenceid=471, filesize=12.0 K 2024-11-11T20:53:32,708 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=154.31 KB/158010 for 759cf260483d400f5615c66d4fd85216 in 1300ms, sequenceid=471, compaction requested=false 2024-11-11T20:53:32,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:32,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:32,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-11T20:53:32,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-11T20:53:32,714 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-11T20:53:32,714 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0720 sec 2024-11-11T20:53:32,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.0780 sec 2024-11-11T20:53:32,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T20:53:32,786 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:53:32,786 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:53:32,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-11T20:53:32,788 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:53:32,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T20:53:32,789 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:53:32,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:53:32,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T20:53:32,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T20:53:32,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:32,944 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 759cf260483d400f5615c66d4fd85216 3/3 column families, dataSize=154.31 KB heapSize=416.63 KB 2024-11-11T20:53:32,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=A 2024-11-11T20:53:32,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:32,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=B 2024-11-11T20:53:32,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:32,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=C 2024-11-11T20:53:32,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:32,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111393e069458b3440db4291ded7a59ac35_759cf260483d400f5615c66d4fd85216 is 50, key is test_row_0/A:col10/1731358411486/Put/seqid=0 2024-11-11T20:53:32,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742016_1192 (size=12454) 2024-11-11T20:53:32,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:32,964 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111393e069458b3440db4291ded7a59ac35_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111393e069458b3440db4291ded7a59ac35_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:32,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/969b14ad256b4fe386a5cc51a791eb5f, store: [table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:32,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/969b14ad256b4fe386a5cc51a791eb5f is 175, key is test_row_0/A:col10/1731358411486/Put/seqid=0 2024-11-11T20:53:32,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742017_1193 (size=31255) 2024-11-11T20:53:33,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T20:53:33,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:33,248 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:33,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:33,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358473265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:33,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:33,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:33,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:33,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:33,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358473376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:33,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:33,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:33,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:33,390 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=507, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/969b14ad256b4fe386a5cc51a791eb5f 2024-11-11T20:53:33,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/9cefcc27e2f4447daf13d45bad2ac56d is 50, key is test_row_0/B:col10/1731358411486/Put/seqid=0 2024-11-11T20:53:33,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742018_1194 (size=12301) 2024-11-11T20:53:33,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T20:53:33,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:33,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358473586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:33,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:33,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:33,588 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:33,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358473645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:33,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358473645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:33,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358473646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:33,649 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_1' in 759cf260483d400f5615c66d4fd85216 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:33,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:33,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:33,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:33,649 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in 759cf260483d400f5615c66d4fd85216 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:33,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:33,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:33,650 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in 759cf260483d400f5615c66d4fd85216 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:33,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:33,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:33,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:33,650 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:33,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:33,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358473661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:33,667 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in 759cf260483d400f5615c66d4fd85216 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:33,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:33,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:33,667 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:33,807 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/9cefcc27e2f4447daf13d45bad2ac56d 2024-11-11T20:53:33,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/e060c6de1bc24bc38b4ffa0b3d17d83b is 50, key is test_row_0/C:col10/1731358411486/Put/seqid=0 2024-11-11T20:53:33,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742019_1195 (size=12301) 2024-11-11T20:53:33,826 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=507 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/e060c6de1bc24bc38b4ffa0b3d17d83b 2024-11-11T20:53:33,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/969b14ad256b4fe386a5cc51a791eb5f as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/969b14ad256b4fe386a5cc51a791eb5f 2024-11-11T20:53:33,846 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/969b14ad256b4fe386a5cc51a791eb5f, entries=150, sequenceid=507, filesize=30.5 K 2024-11-11T20:53:33,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/9cefcc27e2f4447daf13d45bad2ac56d as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/9cefcc27e2f4447daf13d45bad2ac56d 2024-11-11T20:53:33,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,856 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/9cefcc27e2f4447daf13d45bad2ac56d, entries=150, sequenceid=507, filesize=12.0 K 2024-11-11T20:53:33,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/e060c6de1bc24bc38b4ffa0b3d17d83b as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/e060c6de1bc24bc38b4ffa0b3d17d83b 2024-11-11T20:53:33,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,870 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/e060c6de1bc24bc38b4ffa0b3d17d83b, entries=150, sequenceid=507, filesize=12.0 K 2024-11-11T20:53:33,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,873 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~416.58 KB/426576, currentSize=33.54 KB/34350 for 759cf260483d400f5615c66d4fd85216 in 929ms, sequenceid=507, compaction requested=true 2024-11-11T20:53:33,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:33,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:33,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-11T20:53:33,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-11T20:53:33,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,878 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-11T20:53:33,878 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0860 sec 2024-11-11T20:53:33,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,881 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.0930 sec 2024-11-11T20:53:33,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:33,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 759cf260483d400f5615c66d4fd85216 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T20:53:33,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=A 2024-11-11T20:53:33,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:33,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=B 2024-11-11T20:53:33,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:33,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=C 2024-11-11T20:53:33,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:33,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T20:53:33,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,925 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:53:33,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,926 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:53:33,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-11T20:53:33,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T20:53:33,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,929 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:53:33,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,930 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:53:33,930 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:53:33,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c248e4b84b9d4ff78ea00fbaddf28be8_759cf260483d400f5615c66d4fd85216 is 50, key is test_row_0/A:col10/1731358413263/Put/seqid=0 2024-11-11T20:53:33,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742020_1196 (size=9914) 2024-11-11T20:53:33,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:33,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:34,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T20:53:34,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358413993, max timestamp=1731358414050Num uniques 859; 2024-11-11T20:53:34,054 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:53:34,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358413993, max timestamp=1731358414050Num uniques 859; 2024-11-11T20:53:34,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358413993, max timestamp=1731358414050Num uniques 859; 2024-11-11T20:53:34,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:53:34,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:53:34,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T20:53:34,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:34,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:34,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:34,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:34,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:34,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:34,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:34,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358474131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:34,134 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:34,237 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T20:53:34,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:34,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:34,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:34,237 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:34,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:34,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:34,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T20:53:34,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:34,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358474246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:34,249 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:34,351 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:34,355 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c248e4b84b9d4ff78ea00fbaddf28be8_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c248e4b84b9d4ff78ea00fbaddf28be8_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:34,356 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/abfde6b9b31e4f1ca3a139c544ac8a35, store: [table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:34,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/abfde6b9b31e4f1ca3a139c544ac8a35 is 175, key is test_row_0/A:col10/1731358413263/Put/seqid=0 2024-11-11T20:53:34,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742021_1197 (size=22561) 2024-11-11T20:53:34,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T20:53:34,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:34,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:34,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:34,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:34,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:34,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:34,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:34,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358474456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:34,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:34,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:34,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T20:53:34,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:34,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:34,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:34,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:34,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:34,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:34,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T20:53:34,697 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T20:53:34,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:34,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:34,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:34,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:34,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:34,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:34,761 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=517, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/abfde6b9b31e4f1ca3a139c544ac8a35 2024-11-11T20:53:34,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:34,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358474767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:34,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:34,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:34,768 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:34,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/05680956342b48ba96729075663d4049 is 50, key is test_row_0/B:col10/1731358413263/Put/seqid=0 2024-11-11T20:53:34,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742022_1198 (size=9857) 2024-11-11T20:53:34,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T20:53:34,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:34,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:34,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:34,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:34,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:34,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:35,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T20:53:35,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:35,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:35,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:35,003 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:35,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:35,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:35,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T20:53:35,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T20:53:35,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:35,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:35,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:35,158 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:35,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:35,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:35,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/05680956342b48ba96729075663d4049 2024-11-11T20:53:35,190 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/fcab27aa178c4363a40341036d2e69e8 is 50, key is test_row_0/C:col10/1731358413263/Put/seqid=0 2024-11-11T20:53:35,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742023_1199 (size=9857) 2024-11-11T20:53:35,200 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=517 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/fcab27aa178c4363a40341036d2e69e8 2024-11-11T20:53:35,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/abfde6b9b31e4f1ca3a139c544ac8a35 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/abfde6b9b31e4f1ca3a139c544ac8a35 2024-11-11T20:53:35,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/abfde6b9b31e4f1ca3a139c544ac8a35, entries=100, sequenceid=517, filesize=22.0 K 2024-11-11T20:53:35,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/05680956342b48ba96729075663d4049 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/05680956342b48ba96729075663d4049 2024-11-11T20:53:35,226 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/05680956342b48ba96729075663d4049, entries=100, sequenceid=517, filesize=9.6 K 2024-11-11T20:53:35,227 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/fcab27aa178c4363a40341036d2e69e8 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/fcab27aa178c4363a40341036d2e69e8 2024-11-11T20:53:35,231 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/fcab27aa178c4363a40341036d2e69e8, entries=100, sequenceid=517, filesize=9.6 K 2024-11-11T20:53:35,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 759cf260483d400f5615c66d4fd85216 in 1321ms, sequenceid=517, compaction requested=true 2024-11-11T20:53:35,233 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:35,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 759cf260483d400f5615c66d4fd85216:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:53:35,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:35,233 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:53:35,233 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:53:35,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 759cf260483d400f5615c66d4fd85216:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:53:35,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:35,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 759cf260483d400f5615c66d4fd85216:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:53:35,233 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:35,234 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47374 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:53:35,234 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 116940 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:53:35,234 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 759cf260483d400f5615c66d4fd85216/B is initiating minor compaction (all files) 2024-11-11T20:53:35,234 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 759cf260483d400f5615c66d4fd85216/A is initiating minor compaction (all files) 2024-11-11T20:53:35,234 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 759cf260483d400f5615c66d4fd85216/B in TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:35,234 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 759cf260483d400f5615c66d4fd85216/A in TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:35,234 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/35df194388ce48cbb174af19d154bf08, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/fa9038b799c24a1eb0861e87f2fc9de3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/969b14ad256b4fe386a5cc51a791eb5f, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/abfde6b9b31e4f1ca3a139c544ac8a35] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp, totalSize=114.2 K 2024-11-11T20:53:35,234 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/bcb75ea0c1f1409fad465fb9efdaff8b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/0653962dff64429ab9f348544216cf3a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/9cefcc27e2f4447daf13d45bad2ac56d, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/05680956342b48ba96729075663d4049] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp, totalSize=46.3 K 2024-11-11T20:53:35,234 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:35,234 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. files: [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/35df194388ce48cbb174af19d154bf08, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/fa9038b799c24a1eb0861e87f2fc9de3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/969b14ad256b4fe386a5cc51a791eb5f, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/abfde6b9b31e4f1ca3a139c544ac8a35] 2024-11-11T20:53:35,235 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting bcb75ea0c1f1409fad465fb9efdaff8b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731358409962 2024-11-11T20:53:35,235 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35df194388ce48cbb174af19d154bf08, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731358409962 2024-11-11T20:53:35,235 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 0653962dff64429ab9f348544216cf3a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1731358410289 2024-11-11T20:53:35,235 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa9038b799c24a1eb0861e87f2fc9de3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1731358410289 2024-11-11T20:53:35,235 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cefcc27e2f4447daf13d45bad2ac56d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1731358411482 2024-11-11T20:53:35,235 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 969b14ad256b4fe386a5cc51a791eb5f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1731358411482 2024-11-11T20:53:35,236 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 05680956342b48ba96729075663d4049, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1731358413263 2024-11-11T20:53:35,236 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting abfde6b9b31e4f1ca3a139c544ac8a35, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1731358413263 2024-11-11T20:53:35,247 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 759cf260483d400f5615c66d4fd85216#B#compaction#162 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:35,247 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/49c553c628b7403e9f7b5490d9d0feee is 50, key is test_row_0/B:col10/1731358413263/Put/seqid=0 2024-11-11T20:53:35,252 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:35,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742024_1200 (size=13051) 2024-11-11T20:53:35,257 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111162848a56692c4442bc1753f93d02da56_759cf260483d400f5615c66d4fd85216 store=[table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:35,259 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111162848a56692c4442bc1753f93d02da56_759cf260483d400f5615c66d4fd85216, store=[table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:35,260 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111162848a56692c4442bc1753f93d02da56_759cf260483d400f5615c66d4fd85216 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:35,262 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/49c553c628b7403e9f7b5490d9d0feee as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/49c553c628b7403e9f7b5490d9d0feee 2024-11-11T20:53:35,271 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 759cf260483d400f5615c66d4fd85216/B of 759cf260483d400f5615c66d4fd85216 into 49c553c628b7403e9f7b5490d9d0feee(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:35,271 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:35,271 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., storeName=759cf260483d400f5615c66d4fd85216/B, priority=12, startTime=1731358415233; duration=0sec 2024-11-11T20:53:35,271 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:35,271 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 759cf260483d400f5615c66d4fd85216:B 2024-11-11T20:53:35,271 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:53:35,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742025_1201 (size=4469) 2024-11-11T20:53:35,276 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47374 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:53:35,276 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 759cf260483d400f5615c66d4fd85216/C is initiating minor compaction (all files) 2024-11-11T20:53:35,276 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 759cf260483d400f5615c66d4fd85216/C in TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:35,276 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/97c1b4db81bd4685920cb8e15692daea, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/c9bcea4adca84e358a94cde943318653, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/e060c6de1bc24bc38b4ffa0b3d17d83b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/fcab27aa178c4363a40341036d2e69e8] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp, totalSize=46.3 K 2024-11-11T20:53:35,277 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 97c1b4db81bd4685920cb8e15692daea, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=462, earliestPutTs=1731358409962 2024-11-11T20:53:35,277 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting c9bcea4adca84e358a94cde943318653, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=471, earliestPutTs=1731358410289 2024-11-11T20:53:35,277 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 759cf260483d400f5615c66d4fd85216#A#compaction#163 average throughput is 0.98 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:35,277 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting e060c6de1bc24bc38b4ffa0b3d17d83b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=507, earliestPutTs=1731358411482 2024-11-11T20:53:35,278 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting fcab27aa178c4363a40341036d2e69e8, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=517, earliestPutTs=1731358413263 2024-11-11T20:53:35,278 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/b5e3869045f34f0f8d4ea311c986bad3 is 175, key is test_row_0/A:col10/1731358413263/Put/seqid=0 2024-11-11T20:53:35,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:35,281 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 759cf260483d400f5615c66d4fd85216 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-11T20:53:35,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=A 2024-11-11T20:53:35,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:35,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=B 2024-11-11T20:53:35,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:35,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=C 2024-11-11T20:53:35,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:35,297 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 759cf260483d400f5615c66d4fd85216#C#compaction#164 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:35,298 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/067cfeae5f384216aa30c422ddf7c709 is 50, key is test_row_0/C:col10/1731358413263/Put/seqid=0 2024-11-11T20:53:35,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742026_1202 (size=32112) 2024-11-11T20:53:35,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111191bf72f47f544e31b3ccaac50210ee9e_759cf260483d400f5615c66d4fd85216 is 50, key is test_row_0/A:col10/1731358414128/Put/seqid=0 2024-11-11T20:53:35,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742027_1203 (size=13051) 2024-11-11T20:53:35,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T20:53:35,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:35,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:35,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:35,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:35,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:35,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:35,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742028_1204 (size=14994) 2024-11-11T20:53:35,316 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:35,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:35,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358475322, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:35,328 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111191bf72f47f544e31b3ccaac50210ee9e_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111191bf72f47f544e31b3ccaac50210ee9e_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:35,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:35,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:35,329 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:35,330 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/c716c458a22649389b7ad5fd2fc554e7, store: [table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:35,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/c716c458a22649389b7ad5fd2fc554e7 is 175, key is test_row_0/A:col10/1731358414128/Put/seqid=0 2024-11-11T20:53:35,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742029_1205 (size=39949) 2024-11-11T20:53:35,438 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:35,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358475436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:35,438 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:35,465 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T20:53:35,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:35,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:35,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:35,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:35,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:35,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:35,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T20:53:35,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:35,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:35,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:35,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:35,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:35,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:35,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358475645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:35,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:35,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:35,649 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:35,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:35,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49538 deadline: 1731358475655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:35,660 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in 759cf260483d400f5615c66d4fd85216 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:35,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:35,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:35,661 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:35,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:35,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49480 deadline: 1731358475675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:35,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:35,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49536 deadline: 1731358475675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:35,677 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_1' in 759cf260483d400f5615c66d4fd85216 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:35,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:35,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:35,677 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:35,677 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in 759cf260483d400f5615c66d4fd85216 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:35,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:35,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:35,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:35,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:35,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49506 deadline: 1731358475689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:35,694 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in 759cf260483d400f5615c66d4fd85216 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:35,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:35,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:35,695 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:35,704 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/b5e3869045f34f0f8d4ea311c986bad3 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/b5e3869045f34f0f8d4ea311c986bad3 2024-11-11T20:53:35,710 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 759cf260483d400f5615c66d4fd85216/A of 759cf260483d400f5615c66d4fd85216 into b5e3869045f34f0f8d4ea311c986bad3(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:35,710 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:35,710 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., storeName=759cf260483d400f5615c66d4fd85216/A, priority=12, startTime=1731358415233; duration=0sec 2024-11-11T20:53:35,710 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:35,710 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 759cf260483d400f5615c66d4fd85216:A 2024-11-11T20:53:35,716 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/067cfeae5f384216aa30c422ddf7c709 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/067cfeae5f384216aa30c422ddf7c709 2024-11-11T20:53:35,721 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 759cf260483d400f5615c66d4fd85216/C of 759cf260483d400f5615c66d4fd85216 into 067cfeae5f384216aa30c422ddf7c709(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:35,722 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:35,722 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., storeName=759cf260483d400f5615c66d4fd85216/C, priority=12, startTime=1731358415233; duration=0sec 2024-11-11T20:53:35,722 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:35,722 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 759cf260483d400f5615c66d4fd85216:C 2024-11-11T20:53:35,756 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=555, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/c716c458a22649389b7ad5fd2fc554e7 2024-11-11T20:53:35,763 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/acccbe5599e44b8f8c65bef92336a6ca is 50, key is test_row_0/B:col10/1731358414128/Put/seqid=0 2024-11-11T20:53:35,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742030_1206 (size=12301) 2024-11-11T20:53:35,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T20:53:35,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:35,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:35,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:35,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:35,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:35,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:35,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T20:53:35,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:35,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:35,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:35,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:35,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:35,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:35,959 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358475955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:35,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:35,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:35,960 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:35,986 INFO [Thread-589 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-589. 2024-11-11T20:53:35,986 DEBUG [Thread-589 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:53:35,986 DEBUG [Thread-589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:35,986 DEBUG [Thread-589 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:35,986 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:53:35,992 INFO [Thread-591 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-591. 2024-11-11T20:53:35,992 DEBUG [Thread-591 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:53:35,992 INFO [Thread-587 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-587. 2024-11-11T20:53:35,992 DEBUG [Thread-591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:35,992 DEBUG [Thread-587 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:53:35,992 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:35,992 DEBUG [Thread-591 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:35,992 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:53:35,992 DEBUG [Thread-587 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:35,992 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:53:35,993 INFO [Thread-593 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-593. 2024-11-11T20:53:35,993 DEBUG [Thread-593 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:53:35,993 DEBUG [Thread-593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:35,993 DEBUG [Thread-593 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:35,993 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:53:36,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T20:53:36,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T20:53:36,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:36,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:36,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:36,078 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:36,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:36,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:36,174 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/acccbe5599e44b8f8c65bef92336a6ca 2024-11-11T20:53:36,185 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/2f9655cbb7be4d8b89012744b648e2c9 is 50, key is test_row_0/C:col10/1731358414128/Put/seqid=0 2024-11-11T20:53:36,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742031_1207 (size=12301) 2024-11-11T20:53:36,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T20:53:36,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:36,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:36,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:36,232 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:36,232 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:36,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:36,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T20:53:36,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:36,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:36,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:36,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:36,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:36,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:36,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:36,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49478 deadline: 1731358476466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:36,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:36,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:36,468 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:53:36,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T20:53:36,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:36,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:36,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:36,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:36,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:36,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:36,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=555 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/2f9655cbb7be4d8b89012744b648e2c9 2024-11-11T20:53:36,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/c716c458a22649389b7ad5fd2fc554e7 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/c716c458a22649389b7ad5fd2fc554e7 2024-11-11T20:53:36,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/c716c458a22649389b7ad5fd2fc554e7, entries=200, sequenceid=555, filesize=39.0 K 2024-11-11T20:53:36,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/acccbe5599e44b8f8c65bef92336a6ca as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/acccbe5599e44b8f8c65bef92336a6ca 2024-11-11T20:53:36,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/acccbe5599e44b8f8c65bef92336a6ca, entries=150, sequenceid=555, filesize=12.0 K 2024-11-11T20:53:36,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/2f9655cbb7be4d8b89012744b648e2c9 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/2f9655cbb7be4d8b89012744b648e2c9 2024-11-11T20:53:36,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/2f9655cbb7be4d8b89012744b648e2c9, entries=150, sequenceid=555, filesize=12.0 K 2024-11-11T20:53:36,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 759cf260483d400f5615c66d4fd85216 in 1337ms, sequenceid=555, compaction requested=false 2024-11-11T20:53:36,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:36,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T20:53:36,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:36,694 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 759cf260483d400f5615c66d4fd85216 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-11T20:53:36,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=A 2024-11-11T20:53:36,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:36,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=B 2024-11-11T20:53:36,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:36,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=C 2024-11-11T20:53:36,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:36,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115c0f62be5fe54f14be2fedc9e463a5f3_759cf260483d400f5615c66d4fd85216 is 50, key is test_row_0/A:col10/1731358415303/Put/seqid=0 2024-11-11T20:53:36,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742032_1208 (size=12454) 2024-11-11T20:53:37,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:37,117 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115c0f62be5fe54f14be2fedc9e463a5f3_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115c0f62be5fe54f14be2fedc9e463a5f3_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:37,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/5367acac687946f48bdc6a98e1dab2cb, store: [table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:37,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/5367acac687946f48bdc6a98e1dab2cb is 175, key is test_row_0/A:col10/1731358415303/Put/seqid=0 2024-11-11T20:53:37,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742033_1209 (size=31255) 2024-11-11T20:53:37,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:37,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. as already flushing 2024-11-11T20:53:37,478 INFO [Thread-576 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-576. 2024-11-11T20:53:37,478 DEBUG [Thread-576 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:53:37,478 DEBUG [Thread-576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:37,478 DEBUG [Thread-576 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:37,478 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:53:37,524 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=566, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/5367acac687946f48bdc6a98e1dab2cb 2024-11-11T20:53:37,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/06402fd07e194644802e65b4512c555b is 50, key is test_row_0/B:col10/1731358415303/Put/seqid=0 2024-11-11T20:53:37,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742034_1210 (size=12301) 2024-11-11T20:53:37,936 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=566 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/06402fd07e194644802e65b4512c555b 2024-11-11T20:53:37,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/ff0034964d684089911b5bfc5a4654e9 is 50, key is test_row_0/C:col10/1731358415303/Put/seqid=0 2024-11-11T20:53:37,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742035_1211 (size=12301) 2024-11-11T20:53:38,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T20:53:38,348 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=566 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/ff0034964d684089911b5bfc5a4654e9 2024-11-11T20:53:38,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/5367acac687946f48bdc6a98e1dab2cb as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/5367acac687946f48bdc6a98e1dab2cb 2024-11-11T20:53:38,357 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/5367acac687946f48bdc6a98e1dab2cb, entries=150, sequenceid=566, filesize=30.5 K 2024-11-11T20:53:38,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/06402fd07e194644802e65b4512c555b as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/06402fd07e194644802e65b4512c555b 2024-11-11T20:53:38,363 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/06402fd07e194644802e65b4512c555b, entries=150, sequenceid=566, filesize=12.0 K 2024-11-11T20:53:38,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/ff0034964d684089911b5bfc5a4654e9 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/ff0034964d684089911b5bfc5a4654e9 2024-11-11T20:53:38,369 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/ff0034964d684089911b5bfc5a4654e9, entries=150, sequenceid=566, filesize=12.0 K 2024-11-11T20:53:38,370 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=6.71 KB/6870 for 759cf260483d400f5615c66d4fd85216 in 1675ms, sequenceid=566, compaction requested=true 2024-11-11T20:53:38,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 759cf260483d400f5615c66d4fd85216: 2024-11-11T20:53:38,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:38,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-11T20:53:38,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-11T20:53:38,373 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-11T20:53:38,373 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.4410 sec 2024-11-11T20:53:38,374 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 4.4470 sec 2024-11-11T20:53:39,678 INFO [Thread-584 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-584. 2024-11-11T20:53:39,679 DEBUG [Thread-584 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:53:39,679 DEBUG [Thread-584 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:39,679 DEBUG [Thread-584 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:39,679 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:53:39,718 INFO [Thread-578 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-578. 2024-11-11T20:53:39,718 DEBUG [Thread-578 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:53:39,718 DEBUG [Thread-578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:39,718 DEBUG [Thread-578 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:39,718 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:53:39,718 INFO [Thread-580 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-580. 2024-11-11T20:53:39,718 INFO [Thread-582 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-582. 2024-11-11T20:53:39,718 DEBUG [Thread-580 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:53:39,718 DEBUG [Thread-582 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:53:39,718 DEBUG [Thread-580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:39,718 DEBUG [Thread-582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:39,718 DEBUG [Thread-580 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:39,718 DEBUG [Thread-582 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:39,718 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:53:39,719 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:53:40,577 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T20:53:42,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T20:53:42,127 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:53:42,127 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T20:53:42,127 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 138 2024-11-11T20:53:42,127 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-11-11T20:53:42,127 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 126 2024-11-11T20:53:42,127 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 62 2024-11-11T20:53:42,127 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 61 2024-11-11T20:53:42,127 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T20:53:42,127 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2714 2024-11-11T20:53:42,128 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2770 2024-11-11T20:53:42,128 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T20:53:42,128 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1151 2024-11-11T20:53:42,128 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3451 rows 2024-11-11T20:53:42,128 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1148 2024-11-11T20:53:42,128 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3444 rows 2024-11-11T20:53:42,128 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T20:53:42,128 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:53:42,128 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:42,129 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:42,129 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:53:42,130 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T20:53:42,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T20:53:42,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T20:53:42,136 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358422135"}]},"ts":"1731358422135"} 2024-11-11T20:53:42,138 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T20:53:42,138 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T20:53:42,139 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T20:53:42,141 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=759cf260483d400f5615c66d4fd85216, UNASSIGN}] 2024-11-11T20:53:42,143 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=759cf260483d400f5615c66d4fd85216, UNASSIGN 2024-11-11T20:53:42,144 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=48 updating hbase:meta row=759cf260483d400f5615c66d4fd85216, regionState=CLOSING, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:42,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=48, ppid=47, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=759cf260483d400f5615c66d4fd85216, UNASSIGN because future has completed 2024-11-11T20:53:42,145 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T20:53:42,145 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; CloseRegionProcedure 759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066}] 2024-11-11T20:53:42,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T20:53:42,299 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] handler.UnassignRegionHandler(122): Close 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:42,299 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T20:53:42,299 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1722): Closing 759cf260483d400f5615c66d4fd85216, disabling compactions & flushes 2024-11-11T20:53:42,299 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:42,299 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:42,299 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. after waiting 0 ms 2024-11-11T20:53:42,299 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:42,300 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(2902): Flushing 759cf260483d400f5615c66d4fd85216 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T20:53:42,300 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=A 2024-11-11T20:53:42,300 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:42,300 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=B 2024-11-11T20:53:42,301 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:42,301 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 759cf260483d400f5615c66d4fd85216, store=C 2024-11-11T20:53:42,301 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:42,312 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118d67476c3b8748eb8f03ea6f848f6594_759cf260483d400f5615c66d4fd85216 is 50, key is test_row_1/A:col10/1731358419716/Put/seqid=0 2024-11-11T20:53:42,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742036_1212 (size=9914) 2024-11-11T20:53:42,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T20:53:42,717 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:42,721 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118d67476c3b8748eb8f03ea6f848f6594_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118d67476c3b8748eb8f03ea6f848f6594_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:42,722 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/3714e0096af54d8a8e373726df168b36, store: [table=TestAcidGuarantees family=A region=759cf260483d400f5615c66d4fd85216] 2024-11-11T20:53:42,722 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/3714e0096af54d8a8e373726df168b36 is 175, key is test_row_1/A:col10/1731358419716/Put/seqid=0 2024-11-11T20:53:42,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742037_1213 (size=22561) 2024-11-11T20:53:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T20:53:43,126 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=574, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/3714e0096af54d8a8e373726df168b36 2024-11-11T20:53:43,137 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/7351376701e9464c9125f3a01ee5196f is 50, key is test_row_1/B:col10/1731358419716/Put/seqid=0 2024-11-11T20:53:43,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742038_1214 (size=9857) 2024-11-11T20:53:43,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T20:53:43,543 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/7351376701e9464c9125f3a01ee5196f 2024-11-11T20:53:43,555 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/c9b3629dc7ed42cdad1bcd351d4bc7df is 50, key is test_row_1/C:col10/1731358419716/Put/seqid=0 2024-11-11T20:53:43,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742039_1215 (size=9857) 2024-11-11T20:53:43,960 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=574 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/c9b3629dc7ed42cdad1bcd351d4bc7df 2024-11-11T20:53:43,968 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/A/3714e0096af54d8a8e373726df168b36 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/3714e0096af54d8a8e373726df168b36 2024-11-11T20:53:43,975 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/3714e0096af54d8a8e373726df168b36, entries=100, sequenceid=574, filesize=22.0 K 2024-11-11T20:53:43,976 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/B/7351376701e9464c9125f3a01ee5196f as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/7351376701e9464c9125f3a01ee5196f 2024-11-11T20:53:43,981 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/7351376701e9464c9125f3a01ee5196f, entries=100, sequenceid=574, filesize=9.6 K 2024-11-11T20:53:43,982 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/.tmp/C/c9b3629dc7ed42cdad1bcd351d4bc7df as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/c9b3629dc7ed42cdad1bcd351d4bc7df 2024-11-11T20:53:43,987 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/c9b3629dc7ed42cdad1bcd351d4bc7df, entries=100, sequenceid=574, filesize=9.6 K 2024-11-11T20:53:43,988 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 759cf260483d400f5615c66d4fd85216 in 1689ms, sequenceid=574, compaction requested=true 2024-11-11T20:53:43,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/7fcc2f45594d4483a7d7047369d9acf3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/4e4cef9ed1164174ab1a8cb770757052, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/84a3fe575a044054967a917cad0ee78c, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/96247eb32be84156a54279415983741b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/2854ca14b9d54f6193374e9b3c5fc61f, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/5d8dbde7f6134230b786b825663af49c, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/047f0ea3ecc648f09aa8c3832a4cb76a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/116ff75ad6e94b79983223b36092bfce, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/36926a235b224be3b0223482f1d86b65, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/03e6fcd7cf864f128c15c541b4e393f9, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/3d1e86127fc04f4e95d3417ec7148013, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/eca246cada6845ffbe2986bfd8c007d4, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/db71af42ae3e4d949df364e19aee952a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/040aede3841247ec97a2ae5f9a2bdc1a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/d3f656392be5477089aa91d243da823c, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/6b0dfa57297a4132b492635b7a1d4fcf, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/cc319fb3c1c74396bb651bdd64874ed2, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/315c8bfd672747bdbf6e54112a4a5fd0, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/35df194388ce48cbb174af19d154bf08, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/fa9038b799c24a1eb0861e87f2fc9de3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/969b14ad256b4fe386a5cc51a791eb5f, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/abfde6b9b31e4f1ca3a139c544ac8a35] to archive 2024-11-11T20:53:43,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:53:43,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/7fcc2f45594d4483a7d7047369d9acf3 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/7fcc2f45594d4483a7d7047369d9acf3 2024-11-11T20:53:43,994 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/4e4cef9ed1164174ab1a8cb770757052 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/4e4cef9ed1164174ab1a8cb770757052 2024-11-11T20:53:43,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/84a3fe575a044054967a917cad0ee78c to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/84a3fe575a044054967a917cad0ee78c 2024-11-11T20:53:43,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/96247eb32be84156a54279415983741b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/96247eb32be84156a54279415983741b 2024-11-11T20:53:43,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/2854ca14b9d54f6193374e9b3c5fc61f to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/2854ca14b9d54f6193374e9b3c5fc61f 2024-11-11T20:53:44,001 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/5d8dbde7f6134230b786b825663af49c to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/5d8dbde7f6134230b786b825663af49c 2024-11-11T20:53:44,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/047f0ea3ecc648f09aa8c3832a4cb76a to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/047f0ea3ecc648f09aa8c3832a4cb76a 2024-11-11T20:53:44,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/116ff75ad6e94b79983223b36092bfce to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/116ff75ad6e94b79983223b36092bfce 2024-11-11T20:53:44,004 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/36926a235b224be3b0223482f1d86b65 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/36926a235b224be3b0223482f1d86b65 2024-11-11T20:53:44,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/03e6fcd7cf864f128c15c541b4e393f9 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/03e6fcd7cf864f128c15c541b4e393f9 2024-11-11T20:53:44,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/3d1e86127fc04f4e95d3417ec7148013 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/3d1e86127fc04f4e95d3417ec7148013 2024-11-11T20:53:44,007 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/eca246cada6845ffbe2986bfd8c007d4 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/eca246cada6845ffbe2986bfd8c007d4 2024-11-11T20:53:44,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/db71af42ae3e4d949df364e19aee952a to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/db71af42ae3e4d949df364e19aee952a 2024-11-11T20:53:44,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/040aede3841247ec97a2ae5f9a2bdc1a to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/040aede3841247ec97a2ae5f9a2bdc1a 2024-11-11T20:53:44,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/d3f656392be5477089aa91d243da823c to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/d3f656392be5477089aa91d243da823c 2024-11-11T20:53:44,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/6b0dfa57297a4132b492635b7a1d4fcf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/6b0dfa57297a4132b492635b7a1d4fcf 2024-11-11T20:53:44,013 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/cc319fb3c1c74396bb651bdd64874ed2 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/cc319fb3c1c74396bb651bdd64874ed2 2024-11-11T20:53:44,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/315c8bfd672747bdbf6e54112a4a5fd0 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/315c8bfd672747bdbf6e54112a4a5fd0 2024-11-11T20:53:44,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/35df194388ce48cbb174af19d154bf08 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/35df194388ce48cbb174af19d154bf08 2024-11-11T20:53:44,016 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/fa9038b799c24a1eb0861e87f2fc9de3 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/fa9038b799c24a1eb0861e87f2fc9de3 2024-11-11T20:53:44,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/969b14ad256b4fe386a5cc51a791eb5f to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/969b14ad256b4fe386a5cc51a791eb5f 2024-11-11T20:53:44,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/abfde6b9b31e4f1ca3a139c544ac8a35 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/abfde6b9b31e4f1ca3a139c544ac8a35 2024-11-11T20:53:44,021 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/96ec7315683546aeb248f47d79dfbca1, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/499bfab9ad9f4778bbff5166c98611d6, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/e4738c4e5868480e9f400032008cc094, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/fe04eb1c133f4940b361bd7e7dba1f2b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/5a92cdb551d34f1aab1e13e7e0d33ccd, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/b29051c5cbce497b892e15a932019c39, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/3c441bac779c4f6c94fc19d5c932df60, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/e646d233d4ef44d7abcb03b76eaab84a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/f612d51028204073a0dae33a07b11303, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/e16338392d45432b853e94df2a3cf995, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/963b7f2537c2451c9cb3ddd153628587, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/fbde519ed52c47a0bf68798c6ad2f09e, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/5122080a859c4167a6611507111d6d0f, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/33d248ba49ff4f01ad743510b55744b2, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/1cb7a5ba45aa49c3946f5b0a17ef84f1, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/29d8d6f78c7c401f8ec7b9c4a6282a19, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/b2b4bb68aae5427a845ce05ac255e23a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/bcb75ea0c1f1409fad465fb9efdaff8b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/bff4dd52e31e40a5bcd3bb792e079ff3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/0653962dff64429ab9f348544216cf3a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/9cefcc27e2f4447daf13d45bad2ac56d, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/05680956342b48ba96729075663d4049] to archive 2024-11-11T20:53:44,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:53:44,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/96ec7315683546aeb248f47d79dfbca1 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/96ec7315683546aeb248f47d79dfbca1 2024-11-11T20:53:44,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/499bfab9ad9f4778bbff5166c98611d6 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/499bfab9ad9f4778bbff5166c98611d6 2024-11-11T20:53:44,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/e4738c4e5868480e9f400032008cc094 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/e4738c4e5868480e9f400032008cc094 2024-11-11T20:53:44,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/fe04eb1c133f4940b361bd7e7dba1f2b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/fe04eb1c133f4940b361bd7e7dba1f2b 2024-11-11T20:53:44,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/5a92cdb551d34f1aab1e13e7e0d33ccd to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/5a92cdb551d34f1aab1e13e7e0d33ccd 2024-11-11T20:53:44,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/b29051c5cbce497b892e15a932019c39 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/b29051c5cbce497b892e15a932019c39 2024-11-11T20:53:44,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/3c441bac779c4f6c94fc19d5c932df60 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/3c441bac779c4f6c94fc19d5c932df60 2024-11-11T20:53:44,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/e646d233d4ef44d7abcb03b76eaab84a to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/e646d233d4ef44d7abcb03b76eaab84a 2024-11-11T20:53:44,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/f612d51028204073a0dae33a07b11303 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/f612d51028204073a0dae33a07b11303 2024-11-11T20:53:44,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/e16338392d45432b853e94df2a3cf995 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/e16338392d45432b853e94df2a3cf995 2024-11-11T20:53:44,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/963b7f2537c2451c9cb3ddd153628587 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/963b7f2537c2451c9cb3ddd153628587 2024-11-11T20:53:44,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/fbde519ed52c47a0bf68798c6ad2f09e to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/fbde519ed52c47a0bf68798c6ad2f09e 2024-11-11T20:53:44,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/5122080a859c4167a6611507111d6d0f to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/5122080a859c4167a6611507111d6d0f 2024-11-11T20:53:44,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/33d248ba49ff4f01ad743510b55744b2 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/33d248ba49ff4f01ad743510b55744b2 2024-11-11T20:53:44,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/1cb7a5ba45aa49c3946f5b0a17ef84f1 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/1cb7a5ba45aa49c3946f5b0a17ef84f1 2024-11-11T20:53:44,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/29d8d6f78c7c401f8ec7b9c4a6282a19 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/29d8d6f78c7c401f8ec7b9c4a6282a19 2024-11-11T20:53:44,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/b2b4bb68aae5427a845ce05ac255e23a to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/b2b4bb68aae5427a845ce05ac255e23a 2024-11-11T20:53:44,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/bcb75ea0c1f1409fad465fb9efdaff8b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/bcb75ea0c1f1409fad465fb9efdaff8b 2024-11-11T20:53:44,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/bff4dd52e31e40a5bcd3bb792e079ff3 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/bff4dd52e31e40a5bcd3bb792e079ff3 2024-11-11T20:53:44,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/0653962dff64429ab9f348544216cf3a to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/0653962dff64429ab9f348544216cf3a 2024-11-11T20:53:44,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/9cefcc27e2f4447daf13d45bad2ac56d to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/9cefcc27e2f4447daf13d45bad2ac56d 2024-11-11T20:53:44,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/05680956342b48ba96729075663d4049 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/05680956342b48ba96729075663d4049 2024-11-11T20:53:44,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/5b36a1998bcd4514b63fbf1aa05d4e77, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/9a4e5687970747589d2a7e813e558145, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/c39e1ddbb0284e9397c08eb63c8840cf, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/c8fc8c4334bf49db93b5c460315e74f4, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/dfd9e5650e724365be02cd64b027e39b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/bc14a3c8775045cf80c31aaac545c332, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/73bc5d506f2141ee82825bd9a3f5b0b5, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/7125a843d7554e5ab1c3e472ae1a3589, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/383d1611c40746d39e169fc5427bfac1, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/9fc8a1dacfe04b038274bb52be949147, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/4ed07674d4ff444da4057fc0e36478e9, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/651958deb3864f70abda4bddc5675968, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/d91164286e6f4bf6947c8a452a635260, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/07e0cdce823c4d38bf302b9aaec8d836, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/2c5733a553604bb4b009f25de607c90b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/9633ec3769594686a19a974599e23545, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/9dadf913473e4bb49911755e3b1972dc, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/97c1b4db81bd4685920cb8e15692daea, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/851253dc500e49dcb9de3259e3e35b7c, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/c9bcea4adca84e358a94cde943318653, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/e060c6de1bc24bc38b4ffa0b3d17d83b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/fcab27aa178c4363a40341036d2e69e8] to archive 2024-11-11T20:53:44,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:53:44,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/5b36a1998bcd4514b63fbf1aa05d4e77 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/5b36a1998bcd4514b63fbf1aa05d4e77 2024-11-11T20:53:44,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/9a4e5687970747589d2a7e813e558145 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/9a4e5687970747589d2a7e813e558145 2024-11-11T20:53:44,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/c39e1ddbb0284e9397c08eb63c8840cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/c39e1ddbb0284e9397c08eb63c8840cf 2024-11-11T20:53:44,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/c8fc8c4334bf49db93b5c460315e74f4 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/c8fc8c4334bf49db93b5c460315e74f4 2024-11-11T20:53:44,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/dfd9e5650e724365be02cd64b027e39b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/dfd9e5650e724365be02cd64b027e39b 2024-11-11T20:53:44,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/bc14a3c8775045cf80c31aaac545c332 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/bc14a3c8775045cf80c31aaac545c332 2024-11-11T20:53:44,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/73bc5d506f2141ee82825bd9a3f5b0b5 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/73bc5d506f2141ee82825bd9a3f5b0b5 2024-11-11T20:53:44,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/7125a843d7554e5ab1c3e472ae1a3589 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/7125a843d7554e5ab1c3e472ae1a3589 2024-11-11T20:53:44,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/383d1611c40746d39e169fc5427bfac1 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/383d1611c40746d39e169fc5427bfac1 2024-11-11T20:53:44,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/9fc8a1dacfe04b038274bb52be949147 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/9fc8a1dacfe04b038274bb52be949147 2024-11-11T20:53:44,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/4ed07674d4ff444da4057fc0e36478e9 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/4ed07674d4ff444da4057fc0e36478e9 2024-11-11T20:53:44,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/651958deb3864f70abda4bddc5675968 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/651958deb3864f70abda4bddc5675968 2024-11-11T20:53:44,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/d91164286e6f4bf6947c8a452a635260 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/d91164286e6f4bf6947c8a452a635260 2024-11-11T20:53:44,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/07e0cdce823c4d38bf302b9aaec8d836 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/07e0cdce823c4d38bf302b9aaec8d836 2024-11-11T20:53:44,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/2c5733a553604bb4b009f25de607c90b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/2c5733a553604bb4b009f25de607c90b 2024-11-11T20:53:44,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/9633ec3769594686a19a974599e23545 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/9633ec3769594686a19a974599e23545 2024-11-11T20:53:44,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/9dadf913473e4bb49911755e3b1972dc to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/9dadf913473e4bb49911755e3b1972dc 2024-11-11T20:53:44,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/97c1b4db81bd4685920cb8e15692daea to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/97c1b4db81bd4685920cb8e15692daea 2024-11-11T20:53:44,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/851253dc500e49dcb9de3259e3e35b7c to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/851253dc500e49dcb9de3259e3e35b7c 2024-11-11T20:53:44,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/c9bcea4adca84e358a94cde943318653 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/c9bcea4adca84e358a94cde943318653 2024-11-11T20:53:44,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/e060c6de1bc24bc38b4ffa0b3d17d83b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/e060c6de1bc24bc38b4ffa0b3d17d83b 2024-11-11T20:53:44,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/fcab27aa178c4363a40341036d2e69e8 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/fcab27aa178c4363a40341036d2e69e8 2024-11-11T20:53:44,089 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/recovered.edits/577.seqid, newMaxSeqId=577, maxSeqId=4 2024-11-11T20:53:44,090 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. 2024-11-11T20:53:44,090 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] regionserver.HRegion(1676): Region close journal for 759cf260483d400f5615c66d4fd85216: Waiting for close lock at 1731358422299Running coprocessor pre-close hooks at 1731358422299Disabling compacts and flushes for region at 1731358422299Disabling writes for close at 1731358422299Obtaining lock to block concurrent updates at 1731358422300 (+1 ms)Preparing flush snapshotting stores in 759cf260483d400f5615c66d4fd85216 at 1731358422300Finished memstore snapshotting TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731358422301 (+1 ms)Flushing stores of TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216. at 1731358422302 (+1 ms)Flushing 759cf260483d400f5615c66d4fd85216/A: creating writer at 1731358422302Flushing mob file 759cf260483d400f5615c66d4fd85216/A: appending metadata at 1731358422312 (+10 ms)Flushing mob file 759cf260483d400f5615c66d4fd85216/A: closing flushed file at 1731358422312Flushing 759cf260483d400f5615c66d4fd85216/A: appending metadata at 1731358422722 (+410 ms)Flushing 759cf260483d400f5615c66d4fd85216/A: closing flushed file at 1731358422722Flushing 759cf260483d400f5615c66d4fd85216/B: creating writer at 1731358423133 (+411 ms)Flushing 759cf260483d400f5615c66d4fd85216/B: appending metadata at 1731358423137 (+4 ms)Flushing 759cf260483d400f5615c66d4fd85216/B: closing flushed file at 1731358423137Flushing 759cf260483d400f5615c66d4fd85216/C: creating writer at 1731358423550 (+413 ms)Flushing 759cf260483d400f5615c66d4fd85216/C: appending metadata at 1731358423554 (+4 ms)Flushing 759cf260483d400f5615c66d4fd85216/C: closing flushed file at 1731358423554Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@66b3f8f9: reopening flushed file at 1731358423967 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3642056: reopening flushed file at 1731358423975 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a1c4672: reopening flushed file at 1731358423981 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 759cf260483d400f5615c66d4fd85216 in 1689ms, sequenceid=574, compaction requested=true at 1731358423988 (+7 ms)Writing region close event to WAL at 1731358424086 (+98 ms)Running coprocessor post-close hooks at 1731358424089 (+3 ms)Closed at 1731358424090 (+1 ms) 2024-11-11T20:53:44,092 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=49}] handler.UnassignRegionHandler(157): Closed 759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:44,092 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=48 updating hbase:meta row=759cf260483d400f5615c66d4fd85216, regionState=CLOSED 2024-11-11T20:53:44,093 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=49, ppid=48, state=RUNNABLE, hasLock=false; CloseRegionProcedure 759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 because future has completed 2024-11-11T20:53:44,095 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-11T20:53:44,095 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; CloseRegionProcedure 759cf260483d400f5615c66d4fd85216, server=51ca66f7ee3c,44669,1731358362066 in 1.9490 sec 2024-11-11T20:53:44,097 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=47 2024-11-11T20:53:44,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=47, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=759cf260483d400f5615c66d4fd85216, UNASSIGN in 1.9540 sec 2024-11-11T20:53:44,100 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-11T20:53:44,100 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9590 sec 2024-11-11T20:53:44,101 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358424101"}]},"ts":"1731358424101"} 2024-11-11T20:53:44,102 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T20:53:44,103 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T20:53:44,104 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9730 sec 2024-11-11T20:53:44,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T20:53:44,286 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:53:44,286 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T20:53:44,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:53:44,289 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=50, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:53:44,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T20:53:44,290 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=50, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:53:44,293 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:44,297 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A, FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B, FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C, FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/recovered.edits] 2024-11-11T20:53:44,300 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/3714e0096af54d8a8e373726df168b36 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/3714e0096af54d8a8e373726df168b36 2024-11-11T20:53:44,302 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/5367acac687946f48bdc6a98e1dab2cb to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/5367acac687946f48bdc6a98e1dab2cb 2024-11-11T20:53:44,304 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/b5e3869045f34f0f8d4ea311c986bad3 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/b5e3869045f34f0f8d4ea311c986bad3 2024-11-11T20:53:44,306 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/c716c458a22649389b7ad5fd2fc554e7 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/A/c716c458a22649389b7ad5fd2fc554e7 2024-11-11T20:53:44,309 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/06402fd07e194644802e65b4512c555b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/06402fd07e194644802e65b4512c555b 2024-11-11T20:53:44,310 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/49c553c628b7403e9f7b5490d9d0feee to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/49c553c628b7403e9f7b5490d9d0feee 2024-11-11T20:53:44,312 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/7351376701e9464c9125f3a01ee5196f to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/7351376701e9464c9125f3a01ee5196f 2024-11-11T20:53:44,313 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/acccbe5599e44b8f8c65bef92336a6ca to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/B/acccbe5599e44b8f8c65bef92336a6ca 2024-11-11T20:53:44,315 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/067cfeae5f384216aa30c422ddf7c709 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/067cfeae5f384216aa30c422ddf7c709 2024-11-11T20:53:44,316 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/2f9655cbb7be4d8b89012744b648e2c9 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/2f9655cbb7be4d8b89012744b648e2c9 2024-11-11T20:53:44,317 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/c9b3629dc7ed42cdad1bcd351d4bc7df to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/c9b3629dc7ed42cdad1bcd351d4bc7df 2024-11-11T20:53:44,319 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/ff0034964d684089911b5bfc5a4654e9 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/C/ff0034964d684089911b5bfc5a4654e9 2024-11-11T20:53:44,322 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/recovered.edits/577.seqid to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216/recovered.edits/577.seqid 2024-11-11T20:53:44,322 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:44,322 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T20:53:44,323 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T20:53:44,324 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T20:53:44,328 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111266cabcf643243bebd090add36423d35_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111266cabcf643243bebd090add36423d35_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:44,330 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111393e069458b3440db4291ded7a59ac35_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111393e069458b3440db4291ded7a59ac35_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:44,331 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111452159d37c374984b3072786f5bac237_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111452159d37c374984b3072786f5bac237_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:44,333 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111589d170cdc354022ab1dcf00d6f3968b_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111589d170cdc354022ab1dcf00d6f3968b_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:44,334 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115c0f62be5fe54f14be2fedc9e463a5f3_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115c0f62be5fe54f14be2fedc9e463a5f3_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:44,335 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111164b36427dd794a3da851c9294524fe70_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111164b36427dd794a3da851c9294524fe70_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:44,336 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111174b163057cd34c198fd96e7a17324fad_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111174b163057cd34c198fd96e7a17324fad_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:44,337 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117915bc0dbe2e49c197e19bd21f4b1b97_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117915bc0dbe2e49c197e19bd21f4b1b97_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:44,338 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117e14dd1cc85b4631a7dbf33a33d53cd4_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117e14dd1cc85b4631a7dbf33a33d53cd4_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:44,339 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111814407abbe1849b59803b7b3a4c5114d_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111814407abbe1849b59803b7b3a4c5114d_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:44,340 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111837775e91d9d4118b2946a29e88f31f1_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111837775e91d9d4118b2946a29e88f31f1_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:44,341 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118d67476c3b8748eb8f03ea6f848f6594_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118d67476c3b8748eb8f03ea6f848f6594_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:44,342 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111191bf72f47f544e31b3ccaac50210ee9e_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111191bf72f47f544e31b3ccaac50210ee9e_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:44,343 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111192ef6e7e8f18472b95bbac85043a5db6_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111192ef6e7e8f18472b95bbac85043a5db6_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:44,344 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111accac9117c7b45cc96852bb31756072a_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111accac9117c7b45cc96852bb31756072a_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:44,345 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ae8cc16fd8664069af46e8f6bec9341a_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ae8cc16fd8664069af46e8f6bec9341a_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:44,347 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b20121690c9546e4b97efa339c1a871a_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b20121690c9546e4b97efa339c1a871a_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:44,348 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b23d0a0a05f8405da56beea940a5e7e7_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b23d0a0a05f8405da56beea940a5e7e7_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:44,349 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c248e4b84b9d4ff78ea00fbaddf28be8_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c248e4b84b9d4ff78ea00fbaddf28be8_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:44,350 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d84a1238b55247c3a6c58e7b1ada70ab_759cf260483d400f5615c66d4fd85216 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d84a1238b55247c3a6c58e7b1ada70ab_759cf260483d400f5615c66d4fd85216 2024-11-11T20:53:44,350 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T20:53:44,353 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=50, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:53:44,355 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T20:53:44,357 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T20:53:44,359 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=50, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:53:44,359 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T20:53:44,359 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731358424359"}]},"ts":"9223372036854775807"} 2024-11-11T20:53:44,361 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T20:53:44,361 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 759cf260483d400f5615c66d4fd85216, NAME => 'TestAcidGuarantees,,1731358392788.759cf260483d400f5615c66d4fd85216.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T20:53:44,361 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T20:53:44,362 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731358424361"}]},"ts":"9223372036854775807"} 2024-11-11T20:53:44,364 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T20:53:44,365 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=50, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:53:44,366 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 79 msec 2024-11-11T20:53:44,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T20:53:44,396 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T20:53:44,396 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:53:44,404 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 250), OpenFileDescriptor=453 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=398 (was 378) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4459 (was 4431) - AvailableMemoryMB LEAK? - 2024-11-11T20:53:44,412 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=398, ProcessCount=11, AvailableMemoryMB=4458 2024-11-11T20:53:44,413 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T20:53:44,413 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T20:53:44,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T20:53:44,415 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T20:53:44,415 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:44,415 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 51 2024-11-11T20:53:44,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-11T20:53:44,416 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T20:53:44,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742040_1216 (size=1061) 2024-11-11T20:53:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-11T20:53:44,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-11T20:53:44,824 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23 2024-11-11T20:53:44,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742041_1217 (size=53) 2024-11-11T20:53:45,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-11T20:53:45,230 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:53:45,230 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 13af503d1b68a9ec8649b1b1c0fa50fb, disabling compactions & flushes 2024-11-11T20:53:45,230 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:45,230 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:45,230 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. after waiting 0 ms 2024-11-11T20:53:45,230 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:45,230 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:45,230 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 13af503d1b68a9ec8649b1b1c0fa50fb: Waiting for close lock at 1731358425230Disabling compacts and flushes for region at 1731358425230Disabling writes for close at 1731358425230Writing region close event to WAL at 1731358425230Closed at 1731358425230 2024-11-11T20:53:45,231 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T20:53:45,232 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731358425231"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358425231"}]},"ts":"1731358425231"} 2024-11-11T20:53:45,233 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T20:53:45,234 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T20:53:45,235 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358425234"}]},"ts":"1731358425234"} 2024-11-11T20:53:45,236 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T20:53:45,236 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13af503d1b68a9ec8649b1b1c0fa50fb, ASSIGN}] 2024-11-11T20:53:45,237 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13af503d1b68a9ec8649b1b1c0fa50fb, ASSIGN 2024-11-11T20:53:45,238 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=13af503d1b68a9ec8649b1b1c0fa50fb, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,44669,1731358362066; forceNewPlan=false, retain=false 2024-11-11T20:53:45,389 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=13af503d1b68a9ec8649b1b1c0fa50fb, regionState=OPENING, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:45,391 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=52, ppid=51, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=13af503d1b68a9ec8649b1b1c0fa50fb, ASSIGN because future has completed 2024-11-11T20:53:45,391 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; OpenRegionProcedure 13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066}] 2024-11-11T20:53:45,546 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:45,546 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7752): Opening region: {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:53:45,546 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:53:45,546 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:53:45,546 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7794): checking encryption for 13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:53:45,546 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(7797): checking classloading for 13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:53:45,548 INFO [StoreOpener-13af503d1b68a9ec8649b1b1c0fa50fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:53:45,549 INFO [StoreOpener-13af503d1b68a9ec8649b1b1c0fa50fb-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T20:53:45,549 INFO [StoreOpener-13af503d1b68a9ec8649b1b1c0fa50fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 13af503d1b68a9ec8649b1b1c0fa50fb columnFamilyName A 2024-11-11T20:53:45,549 DEBUG [StoreOpener-13af503d1b68a9ec8649b1b1c0fa50fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:45,550 INFO [StoreOpener-13af503d1b68a9ec8649b1b1c0fa50fb-1 {}] regionserver.HStore(327): Store=13af503d1b68a9ec8649b1b1c0fa50fb/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:53:45,550 INFO [StoreOpener-13af503d1b68a9ec8649b1b1c0fa50fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:53:45,551 INFO [StoreOpener-13af503d1b68a9ec8649b1b1c0fa50fb-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T20:53:45,551 INFO [StoreOpener-13af503d1b68a9ec8649b1b1c0fa50fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 13af503d1b68a9ec8649b1b1c0fa50fb columnFamilyName B 2024-11-11T20:53:45,552 DEBUG [StoreOpener-13af503d1b68a9ec8649b1b1c0fa50fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:45,552 INFO [StoreOpener-13af503d1b68a9ec8649b1b1c0fa50fb-1 {}] regionserver.HStore(327): Store=13af503d1b68a9ec8649b1b1c0fa50fb/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:53:45,552 INFO [StoreOpener-13af503d1b68a9ec8649b1b1c0fa50fb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:53:45,553 INFO [StoreOpener-13af503d1b68a9ec8649b1b1c0fa50fb-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T20:53:45,553 INFO [StoreOpener-13af503d1b68a9ec8649b1b1c0fa50fb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 13af503d1b68a9ec8649b1b1c0fa50fb columnFamilyName C 2024-11-11T20:53:45,553 DEBUG [StoreOpener-13af503d1b68a9ec8649b1b1c0fa50fb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:53:45,554 INFO [StoreOpener-13af503d1b68a9ec8649b1b1c0fa50fb-1 {}] regionserver.HStore(327): Store=13af503d1b68a9ec8649b1b1c0fa50fb/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:53:45,554 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:45,554 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1038): replaying wal for 13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:53:45,555 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:53:45,555 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:53:45,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-11T20:53:45,556 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1048): stopping wal replay for 13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:53:45,556 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1060): Cleaning up temporary data for 13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:53:45,557 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T20:53:45,558 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1093): writing seq id for 13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:53:45,561 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:53:45,561 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1114): Opened 13af503d1b68a9ec8649b1b1c0fa50fb; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71005725, jitterRate=0.05806775391101837}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T20:53:45,561 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:53:45,562 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegion(1006): Region open journal for 13af503d1b68a9ec8649b1b1c0fa50fb: Running coprocessor pre-open hook at 1731358425546Writing region info on filesystem at 1731358425546Initializing all the Stores at 1731358425547 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358425547Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358425547Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358425547Cleaning up temporary data from old regions at 1731358425556 (+9 ms)Running coprocessor post-open hooks at 1731358425561 (+5 ms)Region opened successfully at 1731358425562 (+1 ms) 2024-11-11T20:53:45,563 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., pid=53, masterSystemTime=1731358425542 2024-11-11T20:53:45,564 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:45,565 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=53}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:45,565 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=52 updating hbase:meta row=13af503d1b68a9ec8649b1b1c0fa50fb, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:45,567 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=53, ppid=52, state=RUNNABLE, hasLock=false; OpenRegionProcedure 13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 because future has completed 2024-11-11T20:53:45,569 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-11T20:53:45,569 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; OpenRegionProcedure 13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 in 176 msec 2024-11-11T20:53:45,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=52, resume processing ppid=51 2024-11-11T20:53:45,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, ppid=51, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13af503d1b68a9ec8649b1b1c0fa50fb, ASSIGN in 333 msec 2024-11-11T20:53:45,571 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T20:53:45,571 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358425571"}]},"ts":"1731358425571"} 2024-11-11T20:53:45,572 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T20:53:45,573 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=51, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T20:53:45,574 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1600 sec 2024-11-11T20:53:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-11-11T20:53:46,566 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:53:46,568 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d8da64d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,568 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:53:46,569 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:53:46,570 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:53:46,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:53:46,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:53:46,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2bf1903d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:53:46,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:53:46,572 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:46,572 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37916, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:53:46,573 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@588d8843, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,573 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:53:46,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:53:46,574 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:53:46,575 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:53:46,576 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T20:53:46,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 51ca66f7ee3c,38429,1731358361462 2024-11-11T20:53:46,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4949afc8 2024-11-11T20:53:46,576 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T20:53:46,577 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37926, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T20:53:46,578 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554689fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,578 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:53:46,578 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:53:46,579 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:53:46,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:53:46,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:53:46,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@212d04c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:53:46,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:53:46,579 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:46,580 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:53:46,581 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fe64140, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,581 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72325ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,581 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:53:46,581 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:53:46,582 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:53:46,582 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:53:46,582 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:53:46,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3360de83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:53:46,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:53:46,583 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:46,584 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:53:46,584 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e4c46c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,585 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6686bc48, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,585 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:53:46,585 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:53:46,586 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:53:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:53:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:53:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c750c61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:53:46,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:53:46,587 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:46,587 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:53:46,588 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c9d0dde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,589 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d70004e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,589 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:53:46,589 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:53:46,590 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:53:46,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:53:46,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:53:46,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@444b4650, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:53:46,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:53:46,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:46,591 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:53:46,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12470667, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,593 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19903550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,593 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:53:46,593 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:53:46,594 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:53:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:53:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:53:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d3160ce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:53:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:53:46,594 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:46,595 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:53:46,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69061557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,597 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b9635b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,597 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:53:46,597 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:53:46,598 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:53:46,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:53:46,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:53:46,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@461025f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:53:46,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:53:46,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:46,599 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:53:46,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d618b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2611025, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,600 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:53:46,600 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:53:46,601 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:53:46,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:53:46,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:53:46,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@788b4983, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:53:46,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:53:46,601 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:46,602 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38066, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:53:46,602 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c8177df, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,603 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8a8ae50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,603 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:53:46,603 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:53:46,604 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:53:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:53:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:53:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10cbf99e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:53:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:53:46,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:46,605 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:53:46,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b40e651, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,606 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@471771d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,606 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:53:46,606 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:53:46,606 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:53:46,607 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:53:46,607 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:53:46,607 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@25f65313, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,607 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:53:46,607 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:53:46,607 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:46,608 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:53:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c752416, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40916ac2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,608 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:53:46,608 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:53:46,609 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:53:46,609 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:53:46,609 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:53:46,609 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@764c0fa4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,609 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:53:46,609 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:53:46,610 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:53:46,610 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38100, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:53:46,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b94728e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:53:46,611 DEBUG [Thread-1043 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:53:46,611 DEBUG [Thread-1032 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:53:46,611 DEBUG [Thread-1037 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:53:46,611 DEBUG [Thread-1028 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:53:46,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:53:46,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:53:46,612 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:53:46,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:53:46,612 DEBUG [Thread-1041 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:53:46,612 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:53:46,612 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:53:46,612 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:53:46,612 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:53:46,612 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:53:46,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:53:46,613 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:53:46,613 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41590, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:53:46,613 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:53:46,613 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41592, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:53:46,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-11T20:53:46,614 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41616, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:53:46,614 DEBUG [Thread-1034 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:53:46,614 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:53:46,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T20:53:46,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:53:46,615 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:53:46,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:53:46,616 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:53:46,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:53:46,616 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:53:46,617 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:53:46,618 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:53:46,618 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:53:46,618 DEBUG [Thread-1045 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:53:46,618 DEBUG [Thread-1039 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:53:46,619 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:53:46,619 DEBUG [Thread-1026 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:53:46,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:53:46,619 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:53:46,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:53:46,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:53:46,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:53:46,620 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:53:46,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:53:46,620 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:53:46,620 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:53:46,621 DEBUG [Thread-1030 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:53:46,621 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:53:46,622 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41662, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:53:46,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:53:46,622 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:53:46,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:53:46,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:53:46,623 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41666, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:53:46,624 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:53:46,626 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:53:46,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:53:46,627 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13af503d1b68a9ec8649b1b1c0fa50fb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T20:53:46,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=A 2024-11-11T20:53:46,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:46,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=B 2024-11-11T20:53:46,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:46,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=C 2024-11-11T20:53:46,627 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:46,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/b6f4e26e5cab4c128950de064ceccdad is 50, key is test_row_1/A:col10/1731358426625/Put/seqid=0 2024-11-11T20:53:46,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358426652, max timestamp=1731358426661Num uniques 859; 2024-11-11T20:53:46,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:53:46,665 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358426652, max timestamp=1731358426661Num uniques 859; 2024-11-11T20:53:46,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:53:46,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358426652, max timestamp=1731358426661Num uniques 859; 2024-11-11T20:53:46,666 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:53:46,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742042_1218 (size=11997) 2024-11-11T20:53:46,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/b6f4e26e5cab4c128950de064ceccdad 2024-11-11T20:53:46,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:46,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41662 deadline: 1731358486687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:46,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:46,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:46,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:46,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:46,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358486688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:46,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:46,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:46,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41666 deadline: 1731358486689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:46,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:46,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:46,694 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:46,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:46,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:46,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358486690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:46,694 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:46,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:46,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:46,694 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:46,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:46,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358486692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:46,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:46,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:46,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:46,699 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/35343a749f114ec9ad4f35a7d72b8e7f is 50, key is test_row_1/B:col10/1731358426625/Put/seqid=0 2024-11-11T20:53:46,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742043_1219 (size=9657) 2024-11-11T20:53:46,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T20:53:46,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T20:53:46,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:46,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:46,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:46,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:46,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:46,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:46,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:46,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41662 deadline: 1731358486796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:46,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:46,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:46,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:46,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358486796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:46,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:46,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:46,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41666 deadline: 1731358486796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:46,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:46,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:46,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358486805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:46,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:46,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:46,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:46,810 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:46,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358486806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:46,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:46,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:46,810 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:46,933 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T20:53:46,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:46,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:46,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:46,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:46,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:46,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:46,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T20:53:47,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:47,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41662 deadline: 1731358487006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:47,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:47,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:47,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358487006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:47,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:47,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41666 deadline: 1731358487006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:47,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:47,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,008 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:47,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:47,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:47,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358487015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:47,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358487015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:47,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:47,020 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:47,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T20:53:47,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:47,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:47,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:47,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:47,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:47,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:47,111 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/35343a749f114ec9ad4f35a7d72b8e7f 2024-11-11T20:53:47,136 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/13cd8c1afb214ba495a0e16eac6f3e1f is 50, key is test_row_1/C:col10/1731358426625/Put/seqid=0 2024-11-11T20:53:47,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742044_1220 (size=9657) 2024-11-11T20:53:47,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T20:53:47,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:47,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:47,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:47,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:47,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:47,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:47,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T20:53:47,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:47,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41662 deadline: 1731358487316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:47,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:47,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,318 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:47,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358487316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:47,318 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:47,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:47,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41666 deadline: 1731358487316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:47,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:47,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:47,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358487325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:47,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:47,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358487325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:47,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:47,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,329 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:47,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T20:53:47,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:47,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:47,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:47,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:47,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:47,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:47,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T20:53:47,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:47,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:47,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:47,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:47,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:47,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:47,588 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/13cd8c1afb214ba495a0e16eac6f3e1f 2024-11-11T20:53:47,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/b6f4e26e5cab4c128950de064ceccdad as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/b6f4e26e5cab4c128950de064ceccdad 2024-11-11T20:53:47,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/b6f4e26e5cab4c128950de064ceccdad, entries=150, sequenceid=11, filesize=11.7 K 2024-11-11T20:53:47,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/35343a749f114ec9ad4f35a7d72b8e7f as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/35343a749f114ec9ad4f35a7d72b8e7f 2024-11-11T20:53:47,611 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/35343a749f114ec9ad4f35a7d72b8e7f, entries=100, sequenceid=11, filesize=9.4 K 2024-11-11T20:53:47,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/13cd8c1afb214ba495a0e16eac6f3e1f as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/13cd8c1afb214ba495a0e16eac6f3e1f 2024-11-11T20:53:47,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/13cd8c1afb214ba495a0e16eac6f3e1f, entries=100, sequenceid=11, filesize=9.4 K 2024-11-11T20:53:47,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 13af503d1b68a9ec8649b1b1c0fa50fb in 991ms, sequenceid=11, compaction requested=false 2024-11-11T20:53:47,619 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T20:53:47,619 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:53:47,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-11T20:53:47,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:47,701 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 13af503d1b68a9ec8649b1b1c0fa50fb 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-11T20:53:47,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=A 2024-11-11T20:53:47,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:47,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=B 2024-11-11T20:53:47,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:47,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=C 2024-11-11T20:53:47,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:47,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/e419c0ba9e044fca9e944b778ba0caea is 50, key is test_row_0/A:col10/1731358426688/Put/seqid=0 2024-11-11T20:53:47,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742045_1221 (size=12001) 2024-11-11T20:53:47,712 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/e419c0ba9e044fca9e944b778ba0caea 2024-11-11T20:53:47,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/1e71ef14678645d1bc2ee0bd409a26d7 is 50, key is test_row_0/B:col10/1731358426688/Put/seqid=0 2024-11-11T20:53:47,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742046_1222 (size=12001) 2024-11-11T20:53:47,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T20:53:47,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:53:47,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:47,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:47,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358487835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:47,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:47,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358487835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:47,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:47,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:47,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41666 deadline: 1731358487836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:47,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,840 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:47,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:47,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358487837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:47,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:47,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,843 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:47,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41662 deadline: 1731358487837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:47,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:47,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:47,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41666 deadline: 1731358487946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:47,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:47,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,948 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:47,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358487946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:47,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:47,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41662 deadline: 1731358487946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:47,949 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:47,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:47,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:48,144 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/1e71ef14678645d1bc2ee0bd409a26d7 2024-11-11T20:53:48,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/8bdc12c85b054e26981d0861354ea4a4 is 50, key is test_row_0/C:col10/1731358426688/Put/seqid=0 2024-11-11T20:53:48,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742047_1223 (size=12001) 2024-11-11T20:53:48,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41666 deadline: 1731358488155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:48,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:48,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:48,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:48,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358488155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:48,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:48,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:48,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:48,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:48,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41662 deadline: 1731358488155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:48,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:48,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:48,160 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:48,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:48,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358488465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:48,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:48,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:48,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:48,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:48,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41662 deadline: 1731358488465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:48,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:48,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:48,468 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:48,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:48,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41666 deadline: 1731358488466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:48,556 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=47 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/8bdc12c85b054e26981d0861354ea4a4 2024-11-11T20:53:48,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/e419c0ba9e044fca9e944b778ba0caea as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/e419c0ba9e044fca9e944b778ba0caea 2024-11-11T20:53:48,563 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/e419c0ba9e044fca9e944b778ba0caea, entries=150, sequenceid=47, filesize=11.7 K 2024-11-11T20:53:48,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/1e71ef14678645d1bc2ee0bd409a26d7 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/1e71ef14678645d1bc2ee0bd409a26d7 2024-11-11T20:53:48,569 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/1e71ef14678645d1bc2ee0bd409a26d7, entries=150, sequenceid=47, filesize=11.7 K 2024-11-11T20:53:48,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/8bdc12c85b054e26981d0861354ea4a4 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/8bdc12c85b054e26981d0861354ea4a4 2024-11-11T20:53:48,574 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/8bdc12c85b054e26981d0861354ea4a4, entries=150, sequenceid=47, filesize=11.7 K 2024-11-11T20:53:48,575 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 13af503d1b68a9ec8649b1b1c0fa50fb in 874ms, sequenceid=47, compaction requested=false 2024-11-11T20:53:48,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:53:48,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:48,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-11T20:53:48,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-11T20:53:48,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-11T20:53:48,579 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9580 sec 2024-11-11T20:53:48,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.9670 sec 2024-11-11T20:53:48,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-11T20:53:48,766 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:53:48,766 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:53:48,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-11T20:53:48,768 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:53:48,768 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:53:48,768 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:53:48,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T20:53:48,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:53:48,848 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13af503d1b68a9ec8649b1b1c0fa50fb 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T20:53:48,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=A 2024-11-11T20:53:48,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:48,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=B 2024-11-11T20:53:48,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:48,848 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=C 2024-11-11T20:53:48,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:48,853 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/f9d10c6348734859a98e2030b0493db1 is 50, key is test_row_0/A:col10/1731358428847/Put/seqid=0 2024-11-11T20:53:48,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742048_1224 (size=11997) 2024-11-11T20:53:48,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T20:53:48,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358428879, max timestamp=1731358428889Num uniques 859; 2024-11-11T20:53:48,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358428879, max timestamp=1731358428889Num uniques 859; 2024-11-11T20:53:48,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:53:48,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:53:48,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358428879, max timestamp=1731358428889Num uniques 859; 2024-11-11T20:53:48,894 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:53:48,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T20:53:48,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:48,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:48,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:48,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:48,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:48,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:48,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:48,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358488926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:48,931 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:48,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:48,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358488931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:48,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:48,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:48,935 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:48,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358488976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:48,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:48,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41662 deadline: 1731358488976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:48,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41666 deadline: 1731358488976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:48,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:48,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:48,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:48,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:48,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:48,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:49,036 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:49,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358489035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:49,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:49,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:49,036 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:49,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:49,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358489046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:49,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:49,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:49,050 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:49,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T20:53:49,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:49,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:49,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:49,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:49,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:49,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:49,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T20:53:49,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T20:53:49,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:49,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:49,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:49,228 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:49,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:49,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:49,249 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:49,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358489245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:49,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:49,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:49,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:49,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/f9d10c6348734859a98e2030b0493db1 2024-11-11T20:53:49,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:49,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358489256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:49,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:49,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:49,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:49,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/b918f482991d4496b79488306eba2918 is 50, key is test_row_0/B:col10/1731358428847/Put/seqid=0 2024-11-11T20:53:49,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742049_1225 (size=9657) 2024-11-11T20:53:49,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T20:53:49,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:49,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:49,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:49,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:49,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:49,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:49,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T20:53:49,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T20:53:49,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:49,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:49,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:49,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:49,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:49,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:49,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:49,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358489555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:49,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:49,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:49,558 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:49,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358489565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:49,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:49,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:49,567 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:49,614 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T20:53:49,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/b918f482991d4496b79488306eba2918 2024-11-11T20:53:49,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/3be8eb6ee08d40a398646a7f9e937791 is 50, key is test_row_0/C:col10/1731358428847/Put/seqid=0 2024-11-11T20:53:49,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742050_1226 (size=9657) 2024-11-11T20:53:49,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T20:53:49,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:49,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:49,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:49,687 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:49,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:49,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T20:53:49,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:49,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:49,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:49,840 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:49,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:49,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T20:53:49,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:49,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358489986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:49,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:49,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41666 deadline: 1731358489986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:49,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:49,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41662 deadline: 1731358489986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:49,988 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_1' in 13af503d1b68a9ec8649b1b1c0fa50fb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2150 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:49,988 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in 13af503d1b68a9ec8649b1b1c0fa50fb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:49,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:49,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:49,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:49,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:49,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:49,988 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:49,988 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_1' in 13af503d1b68a9ec8649b1b1c0fa50fb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:49,989 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:49,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T20:53:49,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:49,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:49,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:49,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:49,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:49,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:50,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358490066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:50,068 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:50,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:50,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358490076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:50,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:50,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:50,079 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:50,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=58 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/3be8eb6ee08d40a398646a7f9e937791 2024-11-11T20:53:50,090 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/f9d10c6348734859a98e2030b0493db1 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/f9d10c6348734859a98e2030b0493db1 2024-11-11T20:53:50,094 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/f9d10c6348734859a98e2030b0493db1, entries=150, sequenceid=58, filesize=11.7 K 2024-11-11T20:53:50,095 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/b918f482991d4496b79488306eba2918 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/b918f482991d4496b79488306eba2918 2024-11-11T20:53:50,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/b918f482991d4496b79488306eba2918, entries=100, sequenceid=58, filesize=9.4 K 2024-11-11T20:53:50,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/3be8eb6ee08d40a398646a7f9e937791 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/3be8eb6ee08d40a398646a7f9e937791 2024-11-11T20:53:50,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/3be8eb6ee08d40a398646a7f9e937791, entries=100, sequenceid=58, filesize=9.4 K 2024-11-11T20:53:50,104 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 13af503d1b68a9ec8649b1b1c0fa50fb in 1256ms, sequenceid=58, compaction requested=true 2024-11-11T20:53:50,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:53:50,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13af503d1b68a9ec8649b1b1c0fa50fb:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:53:50,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:50,104 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:50,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13af503d1b68a9ec8649b1b1c0fa50fb:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:53:50,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:50,104 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:50,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13af503d1b68a9ec8649b1b1c0fa50fb:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:53:50,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:50,105 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31315 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:50,105 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35995 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:50,105 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 13af503d1b68a9ec8649b1b1c0fa50fb/A is initiating minor compaction (all files) 2024-11-11T20:53:50,105 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 13af503d1b68a9ec8649b1b1c0fa50fb/B is initiating minor compaction (all files) 2024-11-11T20:53:50,105 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13af503d1b68a9ec8649b1b1c0fa50fb/B in TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:50,105 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13af503d1b68a9ec8649b1b1c0fa50fb/A in TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:50,105 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/35343a749f114ec9ad4f35a7d72b8e7f, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/1e71ef14678645d1bc2ee0bd409a26d7, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/b918f482991d4496b79488306eba2918] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp, totalSize=30.6 K 2024-11-11T20:53:50,105 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/b6f4e26e5cab4c128950de064ceccdad, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/e419c0ba9e044fca9e944b778ba0caea, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/f9d10c6348734859a98e2030b0493db1] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp, totalSize=35.2 K 2024-11-11T20:53:50,105 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 35343a749f114ec9ad4f35a7d72b8e7f, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731358426623 2024-11-11T20:53:50,105 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6f4e26e5cab4c128950de064ceccdad, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731358426620 2024-11-11T20:53:50,106 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting e419c0ba9e044fca9e944b778ba0caea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731358426688 2024-11-11T20:53:50,106 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e71ef14678645d1bc2ee0bd409a26d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731358426688 2024-11-11T20:53:50,106 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9d10c6348734859a98e2030b0493db1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731358427835 2024-11-11T20:53:50,106 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting b918f482991d4496b79488306eba2918, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731358427836 2024-11-11T20:53:50,115 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13af503d1b68a9ec8649b1b1c0fa50fb#B#compaction#183 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:50,116 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/539f91f460914443bdd320c98cbc3649 is 50, key is test_row_0/B:col10/1731358428847/Put/seqid=0 2024-11-11T20:53:50,116 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13af503d1b68a9ec8649b1b1c0fa50fb#A#compaction#184 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:50,116 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/3207e2065acc413d92693adefe400d95 is 50, key is test_row_0/A:col10/1731358428847/Put/seqid=0 2024-11-11T20:53:50,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742051_1227 (size=12104) 2024-11-11T20:53:50,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742052_1228 (size=12104) 2024-11-11T20:53:50,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-11T20:53:50,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:50,146 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 13af503d1b68a9ec8649b1b1c0fa50fb 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-11T20:53:50,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=A 2024-11-11T20:53:50,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:50,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=B 2024-11-11T20:53:50,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:50,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=C 2024-11-11T20:53:50,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:50,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/92abd2ed15be4bc8af71db574d1eda8c is 50, key is test_row_0/A:col10/1731358428922/Put/seqid=0 2024-11-11T20:53:50,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742053_1229 (size=12001) 2024-11-11T20:53:50,161 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/92abd2ed15be4bc8af71db574d1eda8c 2024-11-11T20:53:50,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/2fd927834087482b9cc3c0975a8c61c1 is 50, key is test_row_0/B:col10/1731358428922/Put/seqid=0 2024-11-11T20:53:50,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742054_1230 (size=12001) 2024-11-11T20:53:50,529 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/3207e2065acc413d92693adefe400d95 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/3207e2065acc413d92693adefe400d95 2024-11-11T20:53:50,530 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/539f91f460914443bdd320c98cbc3649 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/539f91f460914443bdd320c98cbc3649 2024-11-11T20:53:50,534 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13af503d1b68a9ec8649b1b1c0fa50fb/A of 13af503d1b68a9ec8649b1b1c0fa50fb into 3207e2065acc413d92693adefe400d95(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:50,534 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:53:50,534 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., storeName=13af503d1b68a9ec8649b1b1c0fa50fb/A, priority=13, startTime=1731358430104; duration=0sec 2024-11-11T20:53:50,534 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:50,534 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13af503d1b68a9ec8649b1b1c0fa50fb:A 2024-11-11T20:53:50,534 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:50,535 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13af503d1b68a9ec8649b1b1c0fa50fb/B of 13af503d1b68a9ec8649b1b1c0fa50fb into 539f91f460914443bdd320c98cbc3649(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:50,535 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:53:50,536 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., storeName=13af503d1b68a9ec8649b1b1c0fa50fb/B, priority=13, startTime=1731358430104; duration=0sec 2024-11-11T20:53:50,536 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:50,536 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13af503d1b68a9ec8649b1b1c0fa50fb:B 2024-11-11T20:53:50,536 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 31315 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:50,536 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 13af503d1b68a9ec8649b1b1c0fa50fb/C is initiating minor compaction (all files) 2024-11-11T20:53:50,536 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13af503d1b68a9ec8649b1b1c0fa50fb/C in TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:50,536 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/13cd8c1afb214ba495a0e16eac6f3e1f, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/8bdc12c85b054e26981d0861354ea4a4, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/3be8eb6ee08d40a398646a7f9e937791] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp, totalSize=30.6 K 2024-11-11T20:53:50,537 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13cd8c1afb214ba495a0e16eac6f3e1f, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731358426623 2024-11-11T20:53:50,537 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bdc12c85b054e26981d0861354ea4a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=47, earliestPutTs=1731358426688 2024-11-11T20:53:50,537 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3be8eb6ee08d40a398646a7f9e937791, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731358427836 2024-11-11T20:53:50,552 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13af503d1b68a9ec8649b1b1c0fa50fb#C#compaction#187 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:50,552 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/8e1ac923904543aea28c168fb446acd4 is 50, key is test_row_0/C:col10/1731358428847/Put/seqid=0 2024-11-11T20:53:50,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742055_1231 (size=12104) 2024-11-11T20:53:50,574 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/2fd927834087482b9cc3c0975a8c61c1 2024-11-11T20:53:50,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/63fe0b2a74d44ad0a56cb6ef46e2ce88 is 50, key is test_row_0/C:col10/1731358428922/Put/seqid=0 2024-11-11T20:53:50,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742056_1232 (size=12001) 2024-11-11T20:53:50,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T20:53:50,963 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/8e1ac923904543aea28c168fb446acd4 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/8e1ac923904543aea28c168fb446acd4 2024-11-11T20:53:50,968 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13af503d1b68a9ec8649b1b1c0fa50fb/C of 13af503d1b68a9ec8649b1b1c0fa50fb into 8e1ac923904543aea28c168fb446acd4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:50,968 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:53:50,968 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., storeName=13af503d1b68a9ec8649b1b1c0fa50fb/C, priority=13, startTime=1731358430104; duration=0sec 2024-11-11T20:53:50,968 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:50,968 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13af503d1b68a9ec8649b1b1c0fa50fb:C 2024-11-11T20:53:50,987 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/63fe0b2a74d44ad0a56cb6ef46e2ce88 2024-11-11T20:53:50,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/92abd2ed15be4bc8af71db574d1eda8c as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/92abd2ed15be4bc8af71db574d1eda8c 2024-11-11T20:53:50,995 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/92abd2ed15be4bc8af71db574d1eda8c, entries=150, sequenceid=93, filesize=11.7 K 2024-11-11T20:53:50,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/2fd927834087482b9cc3c0975a8c61c1 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/2fd927834087482b9cc3c0975a8c61c1 2024-11-11T20:53:51,000 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/2fd927834087482b9cc3c0975a8c61c1, entries=150, sequenceid=93, filesize=11.7 K 2024-11-11T20:53:51,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/63fe0b2a74d44ad0a56cb6ef46e2ce88 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/63fe0b2a74d44ad0a56cb6ef46e2ce88 2024-11-11T20:53:51,005 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/63fe0b2a74d44ad0a56cb6ef46e2ce88, entries=150, sequenceid=93, filesize=11.7 K 2024-11-11T20:53:51,006 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=0 B/0 for 13af503d1b68a9ec8649b1b1c0fa50fb in 860ms, sequenceid=93, compaction requested=false 2024-11-11T20:53:51,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:53:51,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:51,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-11T20:53:51,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-11T20:53:51,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-11T20:53:51,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2400 sec 2024-11-11T20:53:51,028 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 2.2440 sec 2024-11-11T20:53:51,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:53:51,098 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13af503d1b68a9ec8649b1b1c0fa50fb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T20:53:51,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=A 2024-11-11T20:53:51,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:51,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=B 2024-11-11T20:53:51,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:51,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=C 2024-11-11T20:53:51,099 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:51,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/f465d0ba9a3e47ce96b32eb640ce5a1d is 50, key is test_row_0/A:col10/1731358431097/Put/seqid=0 2024-11-11T20:53:51,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742057_1233 (size=14341) 2024-11-11T20:53:51,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358431131, max timestamp=1731358431142Num uniques 859; 2024-11-11T20:53:51,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358431131, max timestamp=1731358431142Num uniques 859; 2024-11-11T20:53:51,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358431131, max timestamp=1731358431142Num uniques 859; 2024-11-11T20:53:51,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:53:51,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:53:51,145 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:53:51,191 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:51,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358491185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:51,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:51,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:51,192 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:51,192 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:51,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358491188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:51,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:51,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:51,193 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:51,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358491295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:51,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:51,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358491295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:51,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:51,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:51,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:51,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:51,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358491505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:51,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:51,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358491505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:51,509 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:51,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:51,510 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:51,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/f465d0ba9a3e47ce96b32eb640ce5a1d 2024-11-11T20:53:51,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/17efaaa12a7448b1b708a9099176e595 is 50, key is test_row_0/B:col10/1731358431097/Put/seqid=0 2024-11-11T20:53:51,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742058_1234 (size=12001) 2024-11-11T20:53:51,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:51,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358491815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:51,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:51,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358491815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:51,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:51,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:51,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:51,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:51,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:51,821 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:51,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/17efaaa12a7448b1b708a9099176e595 2024-11-11T20:53:51,953 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/6c1da9bb828f45809b1fc12ccd49e206 is 50, key is test_row_0/C:col10/1731358431097/Put/seqid=0 2024-11-11T20:53:51,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742059_1235 (size=12001) 2024-11-11T20:53:52,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:52,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41666 deadline: 1731358491996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:52,001 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in 13af503d1b68a9ec8649b1b1c0fa50fb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:52,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:52,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:52,001 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:52,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:52,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358492015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:52,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:52,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41662 deadline: 1731358492015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:52,019 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_1' in 13af503d1b68a9ec8649b1b1c0fa50fb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:52,019 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_1' in 13af503d1b68a9ec8649b1b1c0fa50fb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:52,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:52,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:52,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:52,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:52,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:52,019 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:52,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358492326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:52,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:52,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:52,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358492326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:52,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:52,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:52,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:52,330 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:52,358 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/6c1da9bb828f45809b1fc12ccd49e206 2024-11-11T20:53:52,362 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/f465d0ba9a3e47ce96b32eb640ce5a1d as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/f465d0ba9a3e47ce96b32eb640ce5a1d 2024-11-11T20:53:52,367 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/f465d0ba9a3e47ce96b32eb640ce5a1d, entries=200, sequenceid=106, filesize=14.0 K 2024-11-11T20:53:52,368 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/17efaaa12a7448b1b708a9099176e595 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/17efaaa12a7448b1b708a9099176e595 2024-11-11T20:53:52,372 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/17efaaa12a7448b1b708a9099176e595, entries=150, sequenceid=106, filesize=11.7 K 2024-11-11T20:53:52,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/6c1da9bb828f45809b1fc12ccd49e206 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/6c1da9bb828f45809b1fc12ccd49e206 2024-11-11T20:53:52,377 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/6c1da9bb828f45809b1fc12ccd49e206, entries=150, sequenceid=106, filesize=11.7 K 2024-11-11T20:53:52,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 13af503d1b68a9ec8649b1b1c0fa50fb in 1280ms, sequenceid=106, compaction requested=true 2024-11-11T20:53:52,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:53:52,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13af503d1b68a9ec8649b1b1c0fa50fb:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:53:52,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:52,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13af503d1b68a9ec8649b1b1c0fa50fb:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:53:52,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:52,378 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:52,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13af503d1b68a9ec8649b1b1c0fa50fb:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:53:52,378 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:52,378 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:52,379 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:52,379 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:52,379 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 13af503d1b68a9ec8649b1b1c0fa50fb/A is initiating minor compaction (all files) 2024-11-11T20:53:52,379 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 13af503d1b68a9ec8649b1b1c0fa50fb/B is initiating minor compaction (all files) 2024-11-11T20:53:52,380 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13af503d1b68a9ec8649b1b1c0fa50fb/B in TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:52,380 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13af503d1b68a9ec8649b1b1c0fa50fb/A in TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:52,380 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/3207e2065acc413d92693adefe400d95, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/92abd2ed15be4bc8af71db574d1eda8c, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/f465d0ba9a3e47ce96b32eb640ce5a1d] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp, totalSize=37.5 K 2024-11-11T20:53:52,380 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/539f91f460914443bdd320c98cbc3649, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/2fd927834087482b9cc3c0975a8c61c1, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/17efaaa12a7448b1b708a9099176e595] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp, totalSize=35.3 K 2024-11-11T20:53:52,380 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 539f91f460914443bdd320c98cbc3649, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731358426690 2024-11-11T20:53:52,380 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3207e2065acc413d92693adefe400d95, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731358426690 2024-11-11T20:53:52,380 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fd927834087482b9cc3c0975a8c61c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731358428922 2024-11-11T20:53:52,381 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92abd2ed15be4bc8af71db574d1eda8c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731358428922 2024-11-11T20:53:52,381 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 17efaaa12a7448b1b708a9099176e595, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731358431092 2024-11-11T20:53:52,381 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting f465d0ba9a3e47ce96b32eb640ce5a1d, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731358431092 2024-11-11T20:53:52,388 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13af503d1b68a9ec8649b1b1c0fa50fb#B#compaction#192 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:52,388 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13af503d1b68a9ec8649b1b1c0fa50fb#A#compaction#193 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:52,388 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/5c6cced84bd548ccbdc2837a460e19c3 is 50, key is test_row_0/B:col10/1731358431097/Put/seqid=0 2024-11-11T20:53:52,388 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/458bedb7ac984de59ca5ace38d3126f7 is 50, key is test_row_0/A:col10/1731358431097/Put/seqid=0 2024-11-11T20:53:52,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742061_1237 (size=12207) 2024-11-11T20:53:52,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742060_1236 (size=12207) 2024-11-11T20:53:52,803 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/458bedb7ac984de59ca5ace38d3126f7 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/458bedb7ac984de59ca5ace38d3126f7 2024-11-11T20:53:52,807 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/5c6cced84bd548ccbdc2837a460e19c3 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/5c6cced84bd548ccbdc2837a460e19c3 2024-11-11T20:53:52,810 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13af503d1b68a9ec8649b1b1c0fa50fb/A of 13af503d1b68a9ec8649b1b1c0fa50fb into 458bedb7ac984de59ca5ace38d3126f7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:52,810 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:53:52,810 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., storeName=13af503d1b68a9ec8649b1b1c0fa50fb/A, priority=13, startTime=1731358432378; duration=0sec 2024-11-11T20:53:52,811 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:52,811 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13af503d1b68a9ec8649b1b1c0fa50fb:A 2024-11-11T20:53:52,811 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:52,812 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:52,812 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 13af503d1b68a9ec8649b1b1c0fa50fb/C is initiating minor compaction (all files) 2024-11-11T20:53:52,812 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13af503d1b68a9ec8649b1b1c0fa50fb/C in TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:52,812 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/8e1ac923904543aea28c168fb446acd4, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/63fe0b2a74d44ad0a56cb6ef46e2ce88, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/6c1da9bb828f45809b1fc12ccd49e206] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp, totalSize=35.3 K 2024-11-11T20:53:52,813 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e1ac923904543aea28c168fb446acd4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=58, earliestPutTs=1731358426690 2024-11-11T20:53:52,813 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63fe0b2a74d44ad0a56cb6ef46e2ce88, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731358428922 2024-11-11T20:53:52,814 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c1da9bb828f45809b1fc12ccd49e206, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731358431092 2024-11-11T20:53:52,814 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13af503d1b68a9ec8649b1b1c0fa50fb/B of 13af503d1b68a9ec8649b1b1c0fa50fb into 5c6cced84bd548ccbdc2837a460e19c3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:52,814 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:53:52,814 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., storeName=13af503d1b68a9ec8649b1b1c0fa50fb/B, priority=13, startTime=1731358432378; duration=0sec 2024-11-11T20:53:52,814 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:52,814 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13af503d1b68a9ec8649b1b1c0fa50fb:B 2024-11-11T20:53:52,820 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13af503d1b68a9ec8649b1b1c0fa50fb#C#compaction#194 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:52,821 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/00ae9a51a4134e6e885ac5b4b933f925 is 50, key is test_row_0/C:col10/1731358431097/Put/seqid=0 2024-11-11T20:53:52,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742062_1238 (size=12207) 2024-11-11T20:53:52,842 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/00ae9a51a4134e6e885ac5b4b933f925 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/00ae9a51a4134e6e885ac5b4b933f925 2024-11-11T20:53:52,850 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13af503d1b68a9ec8649b1b1c0fa50fb/C of 13af503d1b68a9ec8649b1b1c0fa50fb into 00ae9a51a4134e6e885ac5b4b933f925(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:52,850 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:53:52,850 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., storeName=13af503d1b68a9ec8649b1b1c0fa50fb/C, priority=13, startTime=1731358432378; duration=0sec 2024-11-11T20:53:52,851 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:52,851 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13af503d1b68a9ec8649b1b1c0fa50fb:C 2024-11-11T20:53:52,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T20:53:52,926 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:53:52,926 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:53:52,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-11T20:53:52,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T20:53:52,929 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:53:52,929 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:53:52,929 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:53:53,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T20:53:53,081 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-11T20:53:53,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:53,082 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 13af503d1b68a9ec8649b1b1c0fa50fb 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-11T20:53:53,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=A 2024-11-11T20:53:53,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:53,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=B 2024-11-11T20:53:53,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:53,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=C 2024-11-11T20:53:53,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:53,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/9a3edb284f9140c18c8af43406a630b9 is 50, key is test_row_0/A:col10/1731358431179/Put/seqid=0 2024-11-11T20:53:53,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742063_1239 (size=12151) 2024-11-11T20:53:53,090 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/9a3edb284f9140c18c8af43406a630b9 2024-11-11T20:53:53,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/808f7ff9e7da4d1989ed17ec4ab246c3 is 50, key is test_row_0/B:col10/1731358431179/Put/seqid=0 2024-11-11T20:53:53,099 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742064_1240 (size=12151) 2024-11-11T20:53:53,100 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/808f7ff9e7da4d1989ed17ec4ab246c3 2024-11-11T20:53:53,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/43a27c79e08a4a30bc7841bd08f27965 is 50, key is test_row_0/C:col10/1731358431179/Put/seqid=0 2024-11-11T20:53:53,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742065_1241 (size=12151) 2024-11-11T20:53:53,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T20:53:53,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:53:53,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:53,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:53,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358493350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:53,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:53,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:53,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:53,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:53,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358493351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:53,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:53,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:53,355 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:53,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:53,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358493456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:53,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:53,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:53,459 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:53,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:53,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358493465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:53,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:53,539 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=145 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/43a27c79e08a4a30bc7841bd08f27965 2024-11-11T20:53:53,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/9a3edb284f9140c18c8af43406a630b9 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/9a3edb284f9140c18c8af43406a630b9 2024-11-11T20:53:53,549 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/9a3edb284f9140c18c8af43406a630b9, entries=150, sequenceid=145, filesize=11.9 K 2024-11-11T20:53:53,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/808f7ff9e7da4d1989ed17ec4ab246c3 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/808f7ff9e7da4d1989ed17ec4ab246c3 2024-11-11T20:53:53,555 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/808f7ff9e7da4d1989ed17ec4ab246c3, entries=150, sequenceid=145, filesize=11.9 K 2024-11-11T20:53:53,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T20:53:53,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/43a27c79e08a4a30bc7841bd08f27965 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/43a27c79e08a4a30bc7841bd08f27965 2024-11-11T20:53:53,571 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/43a27c79e08a4a30bc7841bd08f27965, entries=150, sequenceid=145, filesize=11.9 K 2024-11-11T20:53:53,572 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 13af503d1b68a9ec8649b1b1c0fa50fb in 491ms, sequenceid=145, compaction requested=false 2024-11-11T20:53:53,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:53:53,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:53,572 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-11T20:53:53,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-11T20:53:53,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-11T20:53:53,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 644 msec 2024-11-11T20:53:53,577 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 649 msec 2024-11-11T20:53:53,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:53:53,668 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13af503d1b68a9ec8649b1b1c0fa50fb 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T20:53:53,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=A 2024-11-11T20:53:53,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:53,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=B 2024-11-11T20:53:53,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:53,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=C 2024-11-11T20:53:53,668 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:53,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/91591f107067456d82dbdcfabde943ad is 50, key is test_row_0/A:col10/1731358433667/Put/seqid=0 2024-11-11T20:53:53,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742066_1242 (size=14541) 2024-11-11T20:53:53,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358433703, max timestamp=1731358433715Num uniques 859; 2024-11-11T20:53:53,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:53:53,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358433703, max timestamp=1731358433715Num uniques 859; 2024-11-11T20:53:53,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:53:53,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358433703, max timestamp=1731358433715Num uniques 859; 2024-11-11T20:53:53,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:53:53,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:53,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358493774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:53,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:53,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:53,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:53,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358493775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:53,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:53,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:53,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:53,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:53,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358493886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:53,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:53,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:53,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:53,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358493886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-11T20:53:54,066 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:53:54,067 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:53:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-11T20:53:54,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T20:53:54,068 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:53:54,069 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:53:54,069 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:53:54,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/91591f107067456d82dbdcfabde943ad 2024-11-11T20:53:54,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/af87063d56fa49dcbb5a905f89f40154 is 50, key is test_row_0/B:col10/1731358433667/Put/seqid=0 2024-11-11T20:53:54,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742067_1243 (size=12151) 2024-11-11T20:53:54,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:54,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358494095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:54,099 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:54,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:54,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358494099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:54,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:54,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:54,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T20:53:54,221 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T20:53:54,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:54,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:54,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:54,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:54,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:54,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T20:53:54,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:54,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:54,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:54,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:54,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:54,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T20:53:54,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358494406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:54,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358494406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:54,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:54,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/af87063d56fa49dcbb5a905f89f40154 2024-11-11T20:53:54,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/1d78e9f86890445bbaf793417d9b625a is 50, key is test_row_0/C:col10/1731358433667/Put/seqid=0 2024-11-11T20:53:54,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742068_1244 (size=12151) 2024-11-11T20:53:54,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T20:53:54,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:54,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:54,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:54,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:54,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:54,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T20:53:54,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:54,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:54,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:54,680 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:54,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T20:53:54,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T20:53:54,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:54,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:54,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:54,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:54,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:54,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/1d78e9f86890445bbaf793417d9b625a 2024-11-11T20:53:54,908 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/91591f107067456d82dbdcfabde943ad as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/91591f107067456d82dbdcfabde943ad 2024-11-11T20:53:54,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/91591f107067456d82dbdcfabde943ad, entries=200, sequenceid=156, filesize=14.2 K 2024-11-11T20:53:54,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/af87063d56fa49dcbb5a905f89f40154 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/af87063d56fa49dcbb5a905f89f40154 2024-11-11T20:53:54,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/af87063d56fa49dcbb5a905f89f40154, entries=150, sequenceid=156, filesize=11.9 K 2024-11-11T20:53:54,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358494916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:54,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358494916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:54,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/1d78e9f86890445bbaf793417d9b625a as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/1d78e9f86890445bbaf793417d9b625a 2024-11-11T20:53:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:54,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/1d78e9f86890445bbaf793417d9b625a, entries=150, sequenceid=156, filesize=11.9 K 2024-11-11T20:53:54,923 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 13af503d1b68a9ec8649b1b1c0fa50fb in 1255ms, sequenceid=156, compaction requested=true 2024-11-11T20:53:54,923 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:53:54,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13af503d1b68a9ec8649b1b1c0fa50fb:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:53:54,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:54,923 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:54,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13af503d1b68a9ec8649b1b1c0fa50fb:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:53:54,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:54,923 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:54,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13af503d1b68a9ec8649b1b1c0fa50fb:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:53:54,923 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:54,924 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:54,924 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:54,924 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 13af503d1b68a9ec8649b1b1c0fa50fb/A is initiating minor compaction (all files) 2024-11-11T20:53:54,924 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 13af503d1b68a9ec8649b1b1c0fa50fb/B is initiating minor compaction (all files) 2024-11-11T20:53:54,924 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13af503d1b68a9ec8649b1b1c0fa50fb/A in TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:54,924 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13af503d1b68a9ec8649b1b1c0fa50fb/B in TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:54,924 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/458bedb7ac984de59ca5ace38d3126f7, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/9a3edb284f9140c18c8af43406a630b9, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/91591f107067456d82dbdcfabde943ad] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp, totalSize=38.0 K 2024-11-11T20:53:54,924 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/5c6cced84bd548ccbdc2837a460e19c3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/808f7ff9e7da4d1989ed17ec4ab246c3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/af87063d56fa49dcbb5a905f89f40154] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp, totalSize=35.7 K 2024-11-11T20:53:54,925 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 458bedb7ac984de59ca5ace38d3126f7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731358431092 2024-11-11T20:53:54,925 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c6cced84bd548ccbdc2837a460e19c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731358431092 2024-11-11T20:53:54,925 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a3edb284f9140c18c8af43406a630b9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731358431179 2024-11-11T20:53:54,925 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 808f7ff9e7da4d1989ed17ec4ab246c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731358431179 2024-11-11T20:53:54,925 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting af87063d56fa49dcbb5a905f89f40154, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731358433343 2024-11-11T20:53:54,925 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91591f107067456d82dbdcfabde943ad, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731358433343 2024-11-11T20:53:54,932 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13af503d1b68a9ec8649b1b1c0fa50fb#A#compaction#201 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:54,933 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/5c18e2bee0444267ad3a9731832cefc5 is 50, key is test_row_0/A:col10/1731358433667/Put/seqid=0 2024-11-11T20:53:54,933 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13af503d1b68a9ec8649b1b1c0fa50fb#B#compaction#202 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:54,934 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/4f951d3af28249c5a1ef77412aa81373 is 50, key is test_row_0/B:col10/1731358433667/Put/seqid=0 2024-11-11T20:53:54,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742069_1245 (size=12459) 2024-11-11T20:53:54,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742070_1246 (size=12459) 2024-11-11T20:53:54,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T20:53:54,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:54,986 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 13af503d1b68a9ec8649b1b1c0fa50fb 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-11T20:53:54,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=A 2024-11-11T20:53:54,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:54,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=B 2024-11-11T20:53:54,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:54,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=C 2024-11-11T20:53:54,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:54,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/6aab9c39fadf42d4bad4d64d99eaf1f0 is 50, key is test_row_0/A:col10/1731358433773/Put/seqid=0 2024-11-11T20:53:54,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742071_1247 (size=12151) 2024-11-11T20:53:55,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T20:53:55,347 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/5c18e2bee0444267ad3a9731832cefc5 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/5c18e2bee0444267ad3a9731832cefc5 2024-11-11T20:53:55,351 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13af503d1b68a9ec8649b1b1c0fa50fb/A of 13af503d1b68a9ec8649b1b1c0fa50fb into 5c18e2bee0444267ad3a9731832cefc5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:55,351 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:53:55,351 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., storeName=13af503d1b68a9ec8649b1b1c0fa50fb/A, priority=13, startTime=1731358434923; duration=0sec 2024-11-11T20:53:55,351 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:55,351 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13af503d1b68a9ec8649b1b1c0fa50fb:A 2024-11-11T20:53:55,351 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:55,353 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36509 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:55,353 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 13af503d1b68a9ec8649b1b1c0fa50fb/C is initiating minor compaction (all files) 2024-11-11T20:53:55,353 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13af503d1b68a9ec8649b1b1c0fa50fb/C in TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:55,353 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/00ae9a51a4134e6e885ac5b4b933f925, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/43a27c79e08a4a30bc7841bd08f27965, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/1d78e9f86890445bbaf793417d9b625a] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp, totalSize=35.7 K 2024-11-11T20:53:55,354 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 00ae9a51a4134e6e885ac5b4b933f925, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731358431092 2024-11-11T20:53:55,354 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43a27c79e08a4a30bc7841bd08f27965, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=145, earliestPutTs=1731358431179 2024-11-11T20:53:55,354 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1d78e9f86890445bbaf793417d9b625a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731358433343 2024-11-11T20:53:55,354 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/4f951d3af28249c5a1ef77412aa81373 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/4f951d3af28249c5a1ef77412aa81373 2024-11-11T20:53:55,362 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13af503d1b68a9ec8649b1b1c0fa50fb/B of 13af503d1b68a9ec8649b1b1c0fa50fb into 4f951d3af28249c5a1ef77412aa81373(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:55,363 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:53:55,363 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., storeName=13af503d1b68a9ec8649b1b1c0fa50fb/B, priority=13, startTime=1731358434923; duration=0sec 2024-11-11T20:53:55,363 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:55,363 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13af503d1b68a9ec8649b1b1c0fa50fb:B 2024-11-11T20:53:55,366 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13af503d1b68a9ec8649b1b1c0fa50fb#C#compaction#204 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:55,367 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/0985038e6ffe470fb081cb09424bb5e9 is 50, key is test_row_0/C:col10/1731358433667/Put/seqid=0 2024-11-11T20:53:55,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742072_1248 (size=12459) 2024-11-11T20:53:55,395 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/6aab9c39fadf42d4bad4d64d99eaf1f0 2024-11-11T20:53:55,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/384d2c4d043649cc9793562aa2ec5b20 is 50, key is test_row_0/B:col10/1731358433773/Put/seqid=0 2024-11-11T20:53:55,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742073_1249 (size=12151) 2024-11-11T20:53:55,775 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/0985038e6ffe470fb081cb09424bb5e9 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/0985038e6ffe470fb081cb09424bb5e9 2024-11-11T20:53:55,779 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13af503d1b68a9ec8649b1b1c0fa50fb/C of 13af503d1b68a9ec8649b1b1c0fa50fb into 0985038e6ffe470fb081cb09424bb5e9(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:55,779 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:53:55,779 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., storeName=13af503d1b68a9ec8649b1b1c0fa50fb/C, priority=13, startTime=1731358434923; duration=0sec 2024-11-11T20:53:55,779 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:55,779 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13af503d1b68a9ec8649b1b1c0fa50fb:C 2024-11-11T20:53:55,808 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/384d2c4d043649cc9793562aa2ec5b20 2024-11-11T20:53:55,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/22929d862d854cbd9006100f1e0819f0 is 50, key is test_row_0/C:col10/1731358433773/Put/seqid=0 2024-11-11T20:53:55,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742074_1250 (size=12151) 2024-11-11T20:53:55,822 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=192 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/22929d862d854cbd9006100f1e0819f0 2024-11-11T20:53:55,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/6aab9c39fadf42d4bad4d64d99eaf1f0 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/6aab9c39fadf42d4bad4d64d99eaf1f0 2024-11-11T20:53:55,831 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/6aab9c39fadf42d4bad4d64d99eaf1f0, entries=150, sequenceid=192, filesize=11.9 K 2024-11-11T20:53:55,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/384d2c4d043649cc9793562aa2ec5b20 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/384d2c4d043649cc9793562aa2ec5b20 2024-11-11T20:53:55,837 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/384d2c4d043649cc9793562aa2ec5b20, entries=150, sequenceid=192, filesize=11.9 K 2024-11-11T20:53:55,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/22929d862d854cbd9006100f1e0819f0 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/22929d862d854cbd9006100f1e0819f0 2024-11-11T20:53:55,843 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/22929d862d854cbd9006100f1e0819f0, entries=150, sequenceid=192, filesize=11.9 K 2024-11-11T20:53:55,844 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=0 B/0 for 13af503d1b68a9ec8649b1b1c0fa50fb in 858ms, sequenceid=192, compaction requested=false 2024-11-11T20:53:55,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:53:55,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:55,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-11T20:53:55,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-11T20:53:55,848 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-11T20:53:55,848 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7770 sec 2024-11-11T20:53:55,850 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 1.7810 sec 2024-11-11T20:53:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:53:55,937 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13af503d1b68a9ec8649b1b1c0fa50fb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T20:53:55,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=A 2024-11-11T20:53:55,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:55,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=B 2024-11-11T20:53:55,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:55,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=C 2024-11-11T20:53:55,938 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:55,947 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/f58c102b3fa94290a3909af3e200c504 is 50, key is test_row_0/A:col10/1731358435936/Put/seqid=0 2024-11-11T20:53:55,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742075_1251 (size=12151) 2024-11-11T20:53:56,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358436007, max timestamp=1731358436027Num uniques 859; 2024-11-11T20:53:56,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:53:56,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358436007, max timestamp=1731358436027Num uniques 859; 2024-11-11T20:53:56,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:53:56,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358436007, max timestamp=1731358436027Num uniques 859; 2024-11-11T20:53:56,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:53:56,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:56,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358496063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:56,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,070 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:56,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731358436051, max timestamp=1731358436069Num uniques 1018; 2024-11-11T20:53:56,072 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-11T20:53:56,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731358436051, max timestamp=1731358436069Num uniques 1018; 2024-11-11T20:53:56,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-11T20:53:56,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=318, cellSize=14558, totalHeapSize=33312, min timestamp=1731358436051, max timestamp=1731358436069Num uniques 1018; 2024-11-11T20:53:56,073 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=14558, suffix heap size=133840,new segment heap size=33312 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=318 2024-11-11T20:53:56,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:56,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358496096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:56,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:56,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358496101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:56,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:56,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41666 deadline: 1731358496101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41662 deadline: 1731358496102, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:56,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:56,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358496175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:56,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,179 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:56,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:56,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358496206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:56,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,209 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:56,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T20:53:56,216 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:53:56,217 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:53:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-11T20:53:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T20:53:56,218 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:53:56,219 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:53:56,219 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:53:56,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358496216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:56,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:56,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41666 deadline: 1731358496216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:56,221 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:56,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,221 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:56,221 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:56,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41662 deadline: 1731358496216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:56,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,221 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:56,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T20:53:56,354 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/f58c102b3fa94290a3909af3e200c504 2024-11-11T20:53:56,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/c64940184bc24d7da56100d603008a1d is 50, key is test_row_0/B:col10/1731358435936/Put/seqid=0 2024-11-11T20:53:56,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742076_1252 (size=12151) 2024-11-11T20:53:56,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T20:53:56,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:56,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:56,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:56,371 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:56,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:56,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:56,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:56,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358496386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:56,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,389 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:56,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:56,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358496415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:56,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,419 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:56,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358496425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:56,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41666 deadline: 1731358496425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,429 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:56,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41662 deadline: 1731358496426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:56,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T20:53:56,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:56,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:56,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:56,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:56,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:56,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:56,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T20:53:56,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T20:53:56,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:56,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:56,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:56,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:56,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:56,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:56,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358496695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:56,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:56,729 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:56,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358496726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:56,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,730 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:56,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:56,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358496736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:56,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:56,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:56,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41666 deadline: 1731358496736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:56,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:56,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41662 deadline: 1731358496736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:56,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:56,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:56,766 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/c64940184bc24d7da56100d603008a1d 2024-11-11T20:53:56,772 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/cbad3b14db944c32a3cb2984ea3df010 is 50, key is test_row_0/C:col10/1731358435936/Put/seqid=0 2024-11-11T20:53:56,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742077_1253 (size=12151) 2024-11-11T20:53:56,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T20:53:56,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:56,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:56,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:56,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:56,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:56,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:56,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T20:53:56,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T20:53:56,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:56,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:56,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:56,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:56,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:57,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T20:53:57,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:57,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:57,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:57,142 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:57,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:57,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:57,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/cbad3b14db944c32a3cb2984ea3df010 2024-11-11T20:53:57,183 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/f58c102b3fa94290a3909af3e200c504 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/f58c102b3fa94290a3909af3e200c504 2024-11-11T20:53:57,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/f58c102b3fa94290a3909af3e200c504, entries=150, sequenceid=205, filesize=11.9 K 2024-11-11T20:53:57,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/c64940184bc24d7da56100d603008a1d as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/c64940184bc24d7da56100d603008a1d 2024-11-11T20:53:57,191 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/c64940184bc24d7da56100d603008a1d, entries=150, sequenceid=205, filesize=11.9 K 2024-11-11T20:53:57,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/cbad3b14db944c32a3cb2984ea3df010 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/cbad3b14db944c32a3cb2984ea3df010 2024-11-11T20:53:57,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/cbad3b14db944c32a3cb2984ea3df010, entries=150, sequenceid=205, filesize=11.9 K 2024-11-11T20:53:57,196 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for 13af503d1b68a9ec8649b1b1c0fa50fb in 1259ms, sequenceid=205, compaction requested=true 2024-11-11T20:53:57,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:53:57,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13af503d1b68a9ec8649b1b1c0fa50fb:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:53:57,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:57,197 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:57,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13af503d1b68a9ec8649b1b1c0fa50fb:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:53:57,197 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:57,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:57,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13af503d1b68a9ec8649b1b1c0fa50fb:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:53:57,197 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:57,197 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:57,197 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:57,198 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 13af503d1b68a9ec8649b1b1c0fa50fb/A is initiating minor compaction (all files) 2024-11-11T20:53:57,198 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 13af503d1b68a9ec8649b1b1c0fa50fb/B is initiating minor compaction (all files) 2024-11-11T20:53:57,198 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13af503d1b68a9ec8649b1b1c0fa50fb/A in TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:57,198 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13af503d1b68a9ec8649b1b1c0fa50fb/B in TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:57,198 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/5c18e2bee0444267ad3a9731832cefc5, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/6aab9c39fadf42d4bad4d64d99eaf1f0, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/f58c102b3fa94290a3909af3e200c504] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp, totalSize=35.9 K 2024-11-11T20:53:57,198 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/4f951d3af28249c5a1ef77412aa81373, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/384d2c4d043649cc9793562aa2ec5b20, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/c64940184bc24d7da56100d603008a1d] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp, totalSize=35.9 K 2024-11-11T20:53:57,198 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 4f951d3af28249c5a1ef77412aa81373, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731358433343 2024-11-11T20:53:57,198 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c18e2bee0444267ad3a9731832cefc5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731358433343 2024-11-11T20:53:57,198 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6aab9c39fadf42d4bad4d64d99eaf1f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731358433738 2024-11-11T20:53:57,198 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 384d2c4d043649cc9793562aa2ec5b20, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731358433738 2024-11-11T20:53:57,198 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting f58c102b3fa94290a3909af3e200c504, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731358435932 2024-11-11T20:53:57,199 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting c64940184bc24d7da56100d603008a1d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731358435932 2024-11-11T20:53:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:53:57,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13af503d1b68a9ec8649b1b1c0fa50fb 3/3 column families, dataSize=154.31 KB heapSize=429.84 KB 2024-11-11T20:53:57,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=A 2024-11-11T20:53:57,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:57,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=B 2024-11-11T20:53:57,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:57,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=C 2024-11-11T20:53:57,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:57,211 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13af503d1b68a9ec8649b1b1c0fa50fb#B#compaction#210 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:57,212 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/b93f3a3176ee4f0b803a2821f2549124 is 50, key is test_row_0/B:col10/1731358435936/Put/seqid=0 2024-11-11T20:53:57,217 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13af503d1b68a9ec8649b1b1c0fa50fb#A#compaction#211 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:57,218 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/cc78b1f15b854d0486106cdb79da5998 is 50, key is test_row_0/A:col10/1731358435936/Put/seqid=0 2024-11-11T20:53:57,219 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/123fd5f1ac62453f9f73b9c435d9ec0b is 50, key is test_row_0/A:col10/1731358436101/Put/seqid=0 2024-11-11T20:53:57,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742078_1254 (size=12561) 2024-11-11T20:53:57,231 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/b93f3a3176ee4f0b803a2821f2549124 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/b93f3a3176ee4f0b803a2821f2549124 2024-11-11T20:53:57,236 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13af503d1b68a9ec8649b1b1c0fa50fb/B of 13af503d1b68a9ec8649b1b1c0fa50fb into b93f3a3176ee4f0b803a2821f2549124(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:57,236 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:53:57,236 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., storeName=13af503d1b68a9ec8649b1b1c0fa50fb/B, priority=13, startTime=1731358437197; duration=0sec 2024-11-11T20:53:57,236 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:53:57,236 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13af503d1b68a9ec8649b1b1c0fa50fb:B 2024-11-11T20:53:57,237 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:53:57,238 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:53:57,238 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 13af503d1b68a9ec8649b1b1c0fa50fb/C is initiating minor compaction (all files) 2024-11-11T20:53:57,238 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13af503d1b68a9ec8649b1b1c0fa50fb/C in TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:57,238 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/0985038e6ffe470fb081cb09424bb5e9, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/22929d862d854cbd9006100f1e0819f0, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/cbad3b14db944c32a3cb2984ea3df010] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp, totalSize=35.9 K 2024-11-11T20:53:57,238 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:57,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358497236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:57,239 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:57,239 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 0985038e6ffe470fb081cb09424bb5e9, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731358433343 2024-11-11T20:53:57,239 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 22929d862d854cbd9006100f1e0819f0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=192, earliestPutTs=1731358433738 2024-11-11T20:53:57,240 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting cbad3b14db944c32a3cb2984ea3df010, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731358435932 2024-11-11T20:53:57,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:57,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358497239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:57,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:57,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:57,244 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:57,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742079_1255 (size=12561) 2024-11-11T20:53:57,249 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358497246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:57,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:57,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41666 deadline: 1731358497246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:57,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:57,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:57,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:57,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:57,250 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:57,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:57,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41662 deadline: 1731358497247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:57,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:57,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:57,252 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13af503d1b68a9ec8649b1b1c0fa50fb#C#compaction#213 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:53:57,252 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/46d60b744bd4444498108f3ce58e1133 is 50, key is test_row_0/C:col10/1731358435936/Put/seqid=0 2024-11-11T20:53:57,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742080_1256 (size=14641) 2024-11-11T20:53:57,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742081_1257 (size=12561) 2024-11-11T20:53:57,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T20:53:57,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:57,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:57,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:57,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:57,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:57,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T20:53:57,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:57,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358497355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:57,358 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:57,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T20:53:57,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:57,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:57,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:57,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:57,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:57,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:57,568 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:57,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358497566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:57,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:57,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:57,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:57,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T20:53:57,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:57,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:57,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:57,601 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:57,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:57,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:57,652 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/cc78b1f15b854d0486106cdb79da5998 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/cc78b1f15b854d0486106cdb79da5998 2024-11-11T20:53:57,658 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13af503d1b68a9ec8649b1b1c0fa50fb/A of 13af503d1b68a9ec8649b1b1c0fa50fb into cc78b1f15b854d0486106cdb79da5998(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:57,658 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:53:57,658 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., storeName=13af503d1b68a9ec8649b1b1c0fa50fb/A, priority=13, startTime=1731358437196; duration=0sec 2024-11-11T20:53:57,658 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:57,658 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13af503d1b68a9ec8649b1b1c0fa50fb:A 2024-11-11T20:53:57,675 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/123fd5f1ac62453f9f73b9c435d9ec0b 2024-11-11T20:53:57,682 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/bf0db16c82364ef397bf19e40fe45e75 is 50, key is test_row_0/B:col10/1731358436101/Put/seqid=0 2024-11-11T20:53:57,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742082_1258 (size=12251) 2024-11-11T20:53:57,700 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/46d60b744bd4444498108f3ce58e1133 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/46d60b744bd4444498108f3ce58e1133 2024-11-11T20:53:57,704 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13af503d1b68a9ec8649b1b1c0fa50fb/C of 13af503d1b68a9ec8649b1b1c0fa50fb into 46d60b744bd4444498108f3ce58e1133(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:53:57,704 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:53:57,704 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., storeName=13af503d1b68a9ec8649b1b1c0fa50fb/C, priority=13, startTime=1731358437197; duration=0sec 2024-11-11T20:53:57,705 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:53:57,705 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13af503d1b68a9ec8649b1b1c0fa50fb:C 2024-11-11T20:53:57,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T20:53:57,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:57,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:57,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:57,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:57,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:57,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:57,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:57,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358497875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:57,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:57,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:57,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:57,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T20:53:57,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:57,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:57,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:57,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:57,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:57,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:58,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T20:53:58,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:58,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:58,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:58,061 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:58,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:58,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:58,088 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/bf0db16c82364ef397bf19e40fe45e75 2024-11-11T20:53:58,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/910b887acc9d4d529350cdbff65b3f27 is 50, key is test_row_0/C:col10/1731358436101/Put/seqid=0 2024-11-11T20:53:58,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742083_1259 (size=12251) 2024-11-11T20:53:58,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T20:53:58,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:58,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:58,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:58,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:58,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:58,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:58,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:58,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358498256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:58,258 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:58,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358498256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:58,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:58,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41662 deadline: 1731358498256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:58,258 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in 13af503d1b68a9ec8649b1b1c0fa50fb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2162 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:58,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:58,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:58,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41666 deadline: 1731358498256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:58,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:58,259 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:58,259 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in 13af503d1b68a9ec8649b1b1c0fa50fb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:58,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:58,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:58,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:58,260 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in 13af503d1b68a9ec8649b1b1c0fa50fb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:58,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:58,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:58,261 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:58,261 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_0' in 13af503d1b68a9ec8649b1b1c0fa50fb of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:53:58,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:58,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:58,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:58,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T20:53:58,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T20:53:58,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:58,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:58,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:58,366 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:58,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:58,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:58,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:53:58,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358498386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:53:58,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:58,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:53:58,388 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:53:58,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/910b887acc9d4d529350cdbff65b3f27 2024-11-11T20:53:58,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/123fd5f1ac62453f9f73b9c435d9ec0b as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/123fd5f1ac62453f9f73b9c435d9ec0b 2024-11-11T20:53:58,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/123fd5f1ac62453f9f73b9c435d9ec0b, entries=200, sequenceid=259, filesize=14.3 K 2024-11-11T20:53:58,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/bf0db16c82364ef397bf19e40fe45e75 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/bf0db16c82364ef397bf19e40fe45e75 2024-11-11T20:53:58,518 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/bf0db16c82364ef397bf19e40fe45e75, entries=150, sequenceid=259, filesize=12.0 K 2024-11-11T20:53:58,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/910b887acc9d4d529350cdbff65b3f27 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/910b887acc9d4d529350cdbff65b3f27 2024-11-11T20:53:58,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T20:53:58,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:58,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:58,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:58,519 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:58,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:58,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:53:58,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/910b887acc9d4d529350cdbff65b3f27, entries=150, sequenceid=259, filesize=12.0 K 2024-11-11T20:53:58,524 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~429.80 KB/440112, currentSize=33.54 KB/34350 for 13af503d1b68a9ec8649b1b1c0fa50fb in 1314ms, sequenceid=259, compaction requested=false 2024-11-11T20:53:58,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:53:58,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T20:53:58,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:58,672 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 13af503d1b68a9ec8649b1b1c0fa50fb 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T20:53:58,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=A 2024-11-11T20:53:58,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:58,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=B 2024-11-11T20:53:58,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:58,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=C 2024-11-11T20:53:58,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:53:58,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/edd313e0c1034f90bd6671847481ac9b is 50, key is test_row_0/A:col10/1731358437225/Put/seqid=0 2024-11-11T20:53:58,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742084_1260 (size=12301) 2024-11-11T20:53:59,081 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/edd313e0c1034f90bd6671847481ac9b 2024-11-11T20:53:59,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/86e89264ac63482abd3f4d02c415bfc8 is 50, key is test_row_0/B:col10/1731358437225/Put/seqid=0 2024-11-11T20:53:59,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742085_1261 (size=12301) 2024-11-11T20:53:59,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:53:59,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:53:59,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358439426, max timestamp=1731358439479Num uniques 859; 2024-11-11T20:53:59,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358439426, max timestamp=1731358439479Num uniques 859; 2024-11-11T20:53:59,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:53:59,481 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:53:59,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358439426, max timestamp=1731358439479Num uniques 859; 2024-11-11T20:53:59,482 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:53:59,507 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/86e89264ac63482abd3f4d02c415bfc8 2024-11-11T20:53:59,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/965232c7188a43c4aec92f3ab4d088ee is 50, key is test_row_0/C:col10/1731358437225/Put/seqid=0 2024-11-11T20:53:59,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742086_1262 (size=12301) 2024-11-11T20:53:59,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731358439531, max timestamp=1731358439571Num uniques 1018; 2024-11-11T20:53:59,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731358439531, max timestamp=1731358439571Num uniques 1018; 2024-11-11T20:53:59,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731358439531, max timestamp=1731358439571Num uniques 1018; 2024-11-11T20:53:59,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T20:53:59,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T20:53:59,575 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T20:53:59,662 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731358439642, max timestamp=1731358439659Num uniques 1027; 2024-11-11T20:53:59,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-11T20:53:59,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731358439642, max timestamp=1731358439659Num uniques 1027; 2024-11-11T20:53:59,663 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-11T20:53:59,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731358439642, max timestamp=1731358439659Num uniques 1027; 2024-11-11T20:53:59,664 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-11T20:53:59,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731358439712, max timestamp=1731358439740Num uniques 1036; 2024-11-11T20:53:59,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731358439712, max timestamp=1731358439740Num uniques 1036; 2024-11-11T20:53:59,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-11T20:53:59,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-11T20:53:59,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731358439712, max timestamp=1731358439740Num uniques 1036; 2024-11-11T20:53:59,744 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47446, new segment data size=8516, suffix heap size=135712,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1036, new segment cells count=186 2024-11-11T20:53:59,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731358439805, max timestamp=1731358439827Num uniques 1045; 2024-11-11T20:53:59,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731358439805, max timestamp=1731358439827Num uniques 1045; 2024-11-11T20:53:59,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-11T20:53:59,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-11T20:53:59,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=195, cellSize=8930, totalHeapSize=20520, min timestamp=1731358439805, max timestamp=1731358439827Num uniques 1045; 2024-11-11T20:53:59,831 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=8930, suffix heap size=136648,new segment heap size=20520 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=195 2024-11-11T20:53:59,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731358439874, max timestamp=1731358439919Num uniques 1054; 2024-11-11T20:53:59,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-11T20:53:59,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731358439874, max timestamp=1731358439919Num uniques 1054; 2024-11-11T20:53:59,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-11T20:53:59,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731358439874, max timestamp=1731358439919Num uniques 1054; 2024-11-11T20:53:59,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48270, new segment data size=7050, suffix heap size=137584,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1054, new segment cells count=154 2024-11-11T20:53:59,931 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/965232c7188a43c4aec92f3ab4d088ee 2024-11-11T20:53:59,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/edd313e0c1034f90bd6671847481ac9b as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/edd313e0c1034f90bd6671847481ac9b 2024-11-11T20:53:59,941 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/edd313e0c1034f90bd6671847481ac9b, entries=150, sequenceid=270, filesize=12.0 K 2024-11-11T20:53:59,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/86e89264ac63482abd3f4d02c415bfc8 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/86e89264ac63482abd3f4d02c415bfc8 2024-11-11T20:53:59,946 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/86e89264ac63482abd3f4d02c415bfc8, entries=150, sequenceid=270, filesize=12.0 K 2024-11-11T20:53:59,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/965232c7188a43c4aec92f3ab4d088ee as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/965232c7188a43c4aec92f3ab4d088ee 2024-11-11T20:53:59,950 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/965232c7188a43c4aec92f3ab4d088ee, entries=150, sequenceid=270, filesize=12.0 K 2024-11-11T20:53:59,951 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=60.38 KB/61830 for 13af503d1b68a9ec8649b1b1c0fa50fb in 1279ms, sequenceid=270, compaction requested=true 2024-11-11T20:53:59,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:53:59,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:53:59,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-11T20:53:59,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-11T20:53:59,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:53:59,954 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13af503d1b68a9ec8649b1b1c0fa50fb 3/3 column families, dataSize=67.09 KB heapSize=186.23 KB 2024-11-11T20:53:59,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=A 2024-11-11T20:53:59,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:59,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=B 2024-11-11T20:53:59,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:59,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=C 2024-11-11T20:53:59,954 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:53:59,955 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-11T20:53:59,955 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.7330 sec 2024-11-11T20:53:59,956 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 3.7380 sec 2024-11-11T20:53:59,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/bf70373a80d341248e85c661dd9a4c4a is 50, key is test_row_0/A:col10/1731358439933/Put/seqid=0 2024-11-11T20:53:59,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742087_1263 (size=14741) 2024-11-11T20:54:00,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358500048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:00,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:00,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:00,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:00,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358500155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:00,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:00,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358500265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:00,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41666 deadline: 1731358500265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:00,268 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in 13af503d1b68a9ec8649b1b1c0fa50fb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:54:00,269 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_0' in 13af503d1b68a9ec8649b1b1c0fa50fb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:54:00,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:00,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:00,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:00,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:00,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:00,269 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:00,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41662 deadline: 1731358500286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:00,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:00,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358500286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:00,290 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in 13af503d1b68a9ec8649b1b1c0fa50fb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:54:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:00,290 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:00,291 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:00,290 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in 13af503d1b68a9ec8649b1b1c0fa50fb of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:54:00,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:00,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:00,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:00,364 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/bf70373a80d341248e85c661dd9a4c4a 2024-11-11T20:54:00,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:00,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358500365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:00,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:00,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:00,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:00,371 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/14790f6ff4344b209fd3e06e60cce7f3 is 50, key is test_row_0/B:col10/1731358439933/Put/seqid=0 2024-11-11T20:54:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T20:54:00,376 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:54:00,376 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:54:00,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-11T20:54:00,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T20:54:00,378 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:54:00,379 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:54:00,379 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:54:00,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742088_1264 (size=12301) 2024-11-11T20:54:00,383 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/14790f6ff4344b209fd3e06e60cce7f3 2024-11-11T20:54:00,389 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/7ec7e7b8999a4f5eaa2b8083c6970965 is 50, key is test_row_0/C:col10/1731358439933/Put/seqid=0 2024-11-11T20:54:00,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742089_1265 (size=12301) 2024-11-11T20:54:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T20:54:00,532 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T20:54:00,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:00,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:54:00,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:00,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:00,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:00,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:00,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:00,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358500675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:00,680 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:00,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:00,681 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:00,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T20:54:00,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:00,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:54:00,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:00,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:00,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:00,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:00,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T20:54:00,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=383 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/7ec7e7b8999a4f5eaa2b8083c6970965 2024-11-11T20:54:00,804 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/bf70373a80d341248e85c661dd9a4c4a as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/bf70373a80d341248e85c661dd9a4c4a 2024-11-11T20:54:00,812 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/bf70373a80d341248e85c661dd9a4c4a, entries=200, sequenceid=383, filesize=14.4 K 2024-11-11T20:54:00,813 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/14790f6ff4344b209fd3e06e60cce7f3 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/14790f6ff4344b209fd3e06e60cce7f3 2024-11-11T20:54:00,819 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/14790f6ff4344b209fd3e06e60cce7f3, entries=150, sequenceid=383, filesize=12.0 K 2024-11-11T20:54:00,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/7ec7e7b8999a4f5eaa2b8083c6970965 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/7ec7e7b8999a4f5eaa2b8083c6970965 2024-11-11T20:54:00,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/7ec7e7b8999a4f5eaa2b8083c6970965, entries=150, sequenceid=383, filesize=12.0 K 2024-11-11T20:54:00,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~186.19 KB/190656, currentSize=114.05 KB/116790 for 13af503d1b68a9ec8649b1b1c0fa50fb in 874ms, sequenceid=383, compaction requested=true 2024-11-11T20:54:00,828 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:54:00,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13af503d1b68a9ec8649b1b1c0fa50fb:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:54:00,828 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:00,828 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:54:00,829 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:54:00,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13af503d1b68a9ec8649b1b1c0fa50fb:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:54:00,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:00,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13af503d1b68a9ec8649b1b1c0fa50fb:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:54:00,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:54:00,832 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49414 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:54:00,832 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54244 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:54:00,832 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 13af503d1b68a9ec8649b1b1c0fa50fb/A is initiating minor compaction (all files) 2024-11-11T20:54:00,832 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 13af503d1b68a9ec8649b1b1c0fa50fb/B is initiating minor compaction (all files) 2024-11-11T20:54:00,832 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13af503d1b68a9ec8649b1b1c0fa50fb/B in TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:00,832 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13af503d1b68a9ec8649b1b1c0fa50fb/A in TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:00,832 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/cc78b1f15b854d0486106cdb79da5998, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/123fd5f1ac62453f9f73b9c435d9ec0b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/edd313e0c1034f90bd6671847481ac9b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/bf70373a80d341248e85c661dd9a4c4a] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp, totalSize=53.0 K 2024-11-11T20:54:00,832 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/b93f3a3176ee4f0b803a2821f2549124, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/bf0db16c82364ef397bf19e40fe45e75, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/86e89264ac63482abd3f4d02c415bfc8, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/14790f6ff4344b209fd3e06e60cce7f3] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp, totalSize=48.3 K 2024-11-11T20:54:00,832 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc78b1f15b854d0486106cdb79da5998, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731358435932 2024-11-11T20:54:00,832 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting b93f3a3176ee4f0b803a2821f2549124, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731358435932 2024-11-11T20:54:00,833 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 123fd5f1ac62453f9f73b9c435d9ec0b, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731358436095 2024-11-11T20:54:00,833 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting bf0db16c82364ef397bf19e40fe45e75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731358436095 2024-11-11T20:54:00,833 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting edd313e0c1034f90bd6671847481ac9b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731358437225 2024-11-11T20:54:00,833 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 86e89264ac63482abd3f4d02c415bfc8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731358437225 2024-11-11T20:54:00,834 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf70373a80d341248e85c661dd9a4c4a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731358439933 2024-11-11T20:54:00,834 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 14790f6ff4344b209fd3e06e60cce7f3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731358439933 2024-11-11T20:54:00,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T20:54:00,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:00,838 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 13af503d1b68a9ec8649b1b1c0fa50fb 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T20:54:00,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=A 2024-11-11T20:54:00,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:00,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=B 2024-11-11T20:54:00,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:00,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=C 2024-11-11T20:54:00,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:00,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/a4d739a19cb741239e46f2f384429c7e is 50, key is test_row_0/A:col10/1731358440039/Put/seqid=0 2024-11-11T20:54:00,849 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13af503d1b68a9ec8649b1b1c0fa50fb#A#compaction#223 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:00,850 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/d24044560e5240eb8a84c3d3d7f75c10 is 50, key is test_row_0/A:col10/1731358439933/Put/seqid=0 2024-11-11T20:54:00,853 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13af503d1b68a9ec8649b1b1c0fa50fb#B#compaction#224 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:00,854 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/7eda733f92a946c2b423fb90433da258 is 50, key is test_row_0/B:col10/1731358439933/Put/seqid=0 2024-11-11T20:54:00,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742091_1267 (size=12301) 2024-11-11T20:54:00,861 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/a4d739a19cb741239e46f2f384429c7e 2024-11-11T20:54:00,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742090_1266 (size=12847) 2024-11-11T20:54:00,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742092_1268 (size=12847) 2024-11-11T20:54:00,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/e56818ee2f76497d9606def4020508d9 is 50, key is test_row_0/B:col10/1731358440039/Put/seqid=0 2024-11-11T20:54:00,877 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/d24044560e5240eb8a84c3d3d7f75c10 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/d24044560e5240eb8a84c3d3d7f75c10 2024-11-11T20:54:00,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742093_1269 (size=12301) 2024-11-11T20:54:00,881 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/e56818ee2f76497d9606def4020508d9 2024-11-11T20:54:00,884 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13af503d1b68a9ec8649b1b1c0fa50fb/A of 13af503d1b68a9ec8649b1b1c0fa50fb into d24044560e5240eb8a84c3d3d7f75c10(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:00,884 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:54:00,884 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., storeName=13af503d1b68a9ec8649b1b1c0fa50fb/A, priority=12, startTime=1731358440828; duration=0sec 2024-11-11T20:54:00,884 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:54:00,884 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13af503d1b68a9ec8649b1b1c0fa50fb:A 2024-11-11T20:54:00,884 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:54:00,885 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49414 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:54:00,886 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 13af503d1b68a9ec8649b1b1c0fa50fb/C is initiating minor compaction (all files) 2024-11-11T20:54:00,886 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13af503d1b68a9ec8649b1b1c0fa50fb/C in TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:00,886 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/46d60b744bd4444498108f3ce58e1133, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/910b887acc9d4d529350cdbff65b3f27, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/965232c7188a43c4aec92f3ab4d088ee, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/7ec7e7b8999a4f5eaa2b8083c6970965] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp, totalSize=48.3 K 2024-11-11T20:54:00,886 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46d60b744bd4444498108f3ce58e1133, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731358435932 2024-11-11T20:54:00,887 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 910b887acc9d4d529350cdbff65b3f27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731358436095 2024-11-11T20:54:00,887 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 965232c7188a43c4aec92f3ab4d088ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731358437225 2024-11-11T20:54:00,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/5e381986204042fdb257d87569f20a44 is 50, key is test_row_0/C:col10/1731358440039/Put/seqid=0 2024-11-11T20:54:00,890 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ec7e7b8999a4f5eaa2b8083c6970965, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731358439933 2024-11-11T20:54:00,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742094_1270 (size=12301) 2024-11-11T20:54:00,897 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/5e381986204042fdb257d87569f20a44 2024-11-11T20:54:00,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/a4d739a19cb741239e46f2f384429c7e as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/a4d739a19cb741239e46f2f384429c7e 2024-11-11T20:54:00,902 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13af503d1b68a9ec8649b1b1c0fa50fb#C#compaction#227 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:00,903 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/b6d96c3d39ef41589ff51629ddf9613b is 50, key is test_row_0/C:col10/1731358439933/Put/seqid=0 2024-11-11T20:54:00,909 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/a4d739a19cb741239e46f2f384429c7e, entries=150, sequenceid=403, filesize=12.0 K 2024-11-11T20:54:00,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/e56818ee2f76497d9606def4020508d9 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/e56818ee2f76497d9606def4020508d9 2024-11-11T20:54:00,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742095_1271 (size=12847) 2024-11-11T20:54:00,916 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/e56818ee2f76497d9606def4020508d9, entries=150, sequenceid=403, filesize=12.0 K 2024-11-11T20:54:00,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/5e381986204042fdb257d87569f20a44 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/5e381986204042fdb257d87569f20a44 2024-11-11T20:54:00,920 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/5e381986204042fdb257d87569f20a44, entries=150, sequenceid=403, filesize=12.0 K 2024-11-11T20:54:00,921 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 13af503d1b68a9ec8649b1b1c0fa50fb in 83ms, sequenceid=403, compaction requested=false 2024-11-11T20:54:00,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:54:00,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:00,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-11T20:54:00,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-11T20:54:00,924 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-11T20:54:00,924 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 543 msec 2024-11-11T20:54:00,926 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 549 msec 2024-11-11T20:54:01,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T20:54:01,006 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:54:01,006 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:54:01,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-11T20:54:01,008 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:54:01,009 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:54:01,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:54:01,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T20:54:01,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T20:54:01,160 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T20:54:01,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:01,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:54:01,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:01,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-11T20:54:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-11T20:54:01,163 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-11T20:54:01,164 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 153 msec 2024-11-11T20:54:01,165 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 158 msec 2024-11-11T20:54:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:54:01,213 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13af503d1b68a9ec8649b1b1c0fa50fb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T20:54:01,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=A 2024-11-11T20:54:01,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:01,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=B 2024-11-11T20:54:01,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:01,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=C 2024-11-11T20:54:01,213 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:01,217 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/15e27f6d871b401496fa168e3ae09cf8 is 50, key is test_row_0/A:col10/1731358441209/Put/seqid=0 2024-11-11T20:54:01,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742096_1272 (size=14741) 2024-11-11T20:54:01,221 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/15e27f6d871b401496fa168e3ae09cf8 2024-11-11T20:54:01,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/54e45cf14a944046a9f5f3e498fc8e72 is 50, key is test_row_0/B:col10/1731358441209/Put/seqid=0 2024-11-11T20:54:01,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742097_1273 (size=12301) 2024-11-11T20:54:01,276 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/7eda733f92a946c2b423fb90433da258 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/7eda733f92a946c2b423fb90433da258 2024-11-11T20:54:01,282 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13af503d1b68a9ec8649b1b1c0fa50fb/B of 13af503d1b68a9ec8649b1b1c0fa50fb into 7eda733f92a946c2b423fb90433da258(size=12.5 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:01,282 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:54:01,282 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., storeName=13af503d1b68a9ec8649b1b1c0fa50fb/B, priority=12, startTime=1731358440828; duration=0sec 2024-11-11T20:54:01,282 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:01,282 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13af503d1b68a9ec8649b1b1c0fa50fb:B 2024-11-11T20:54:01,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358441298, max timestamp=1731358441313Num uniques 859; 2024-11-11T20:54:01,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358441298, max timestamp=1731358441313Num uniques 859; 2024-11-11T20:54:01,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:54:01,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:54:01,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358441298, max timestamp=1731358441313Num uniques 859; 2024-11-11T20:54:01,316 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:54:01,319 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/b6d96c3d39ef41589ff51629ddf9613b as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/b6d96c3d39ef41589ff51629ddf9613b 2024-11-11T20:54:01,323 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13af503d1b68a9ec8649b1b1c0fa50fb/C of 13af503d1b68a9ec8649b1b1c0fa50fb into b6d96c3d39ef41589ff51629ddf9613b(size=12.5 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:01,323 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:54:01,323 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., storeName=13af503d1b68a9ec8649b1b1c0fa50fb/C, priority=12, startTime=1731358440829; duration=0sec 2024-11-11T20:54:01,323 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:01,323 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13af503d1b68a9ec8649b1b1c0fa50fb:C 2024-11-11T20:54:01,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T20:54:01,326 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:54:01,326 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:54:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-11T20:54:01,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T20:54:01,327 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:54:01,328 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:54:01,328 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:54:01,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:01,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358501404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:01,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:01,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T20:54:01,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T20:54:01,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:01,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:54:01,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:01,480 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:01,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:01,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:01,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:01,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 214 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358501516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:01,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:01,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:01,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:01,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T20:54:01,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:01,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:54:01,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:01,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:01,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:01,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:01,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/54e45cf14a944046a9f5f3e498fc8e72 2024-11-11T20:54:01,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T20:54:01,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/6c5cc821764b4c62a9b928f4153f8f86 is 50, key is test_row_0/C:col10/1731358441209/Put/seqid=0 2024-11-11T20:54:01,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742098_1274 (size=12301) 2024-11-11T20:54:01,657 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/6c5cc821764b4c62a9b928f4153f8f86 2024-11-11T20:54:01,661 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/15e27f6d871b401496fa168e3ae09cf8 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/15e27f6d871b401496fa168e3ae09cf8 2024-11-11T20:54:01,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/15e27f6d871b401496fa168e3ae09cf8, entries=200, sequenceid=415, filesize=14.4 K 2024-11-11T20:54:01,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/54e45cf14a944046a9f5f3e498fc8e72 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/54e45cf14a944046a9f5f3e498fc8e72 2024-11-11T20:54:01,672 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/54e45cf14a944046a9f5f3e498fc8e72, entries=150, sequenceid=415, filesize=12.0 K 2024-11-11T20:54:01,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/6c5cc821764b4c62a9b928f4153f8f86 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/6c5cc821764b4c62a9b928f4153f8f86 2024-11-11T20:54:01,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/6c5cc821764b4c62a9b928f4153f8f86, entries=150, sequenceid=415, filesize=12.0 K 2024-11-11T20:54:01,679 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 13af503d1b68a9ec8649b1b1c0fa50fb in 466ms, sequenceid=415, compaction requested=true 2024-11-11T20:54:01,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:54:01,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13af503d1b68a9ec8649b1b1c0fa50fb:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:54:01,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:01,679 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:54:01,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13af503d1b68a9ec8649b1b1c0fa50fb:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:54:01,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:01,679 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:54:01,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13af503d1b68a9ec8649b1b1c0fa50fb:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:54:01,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:54:01,680 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:54:01,680 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:54:01,680 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 13af503d1b68a9ec8649b1b1c0fa50fb/A is initiating minor compaction (all files) 2024-11-11T20:54:01,680 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 13af503d1b68a9ec8649b1b1c0fa50fb/B is initiating minor compaction (all files) 2024-11-11T20:54:01,681 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13af503d1b68a9ec8649b1b1c0fa50fb/B in TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:01,681 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/7eda733f92a946c2b423fb90433da258, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/e56818ee2f76497d9606def4020508d9, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/54e45cf14a944046a9f5f3e498fc8e72] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp, totalSize=36.6 K 2024-11-11T20:54:01,681 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13af503d1b68a9ec8649b1b1c0fa50fb/A in TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:01,681 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/d24044560e5240eb8a84c3d3d7f75c10, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/a4d739a19cb741239e46f2f384429c7e, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/15e27f6d871b401496fa168e3ae09cf8] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp, totalSize=39.0 K 2024-11-11T20:54:01,681 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 7eda733f92a946c2b423fb90433da258, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731358439933 2024-11-11T20:54:01,681 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting d24044560e5240eb8a84c3d3d7f75c10, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731358439933 2024-11-11T20:54:01,681 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting e56818ee2f76497d9606def4020508d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731358440021 2024-11-11T20:54:01,682 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4d739a19cb741239e46f2f384429c7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731358440021 2024-11-11T20:54:01,682 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 54e45cf14a944046a9f5f3e498fc8e72, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731358441201 2024-11-11T20:54:01,682 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15e27f6d871b401496fa168e3ae09cf8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731358441201 2024-11-11T20:54:01,692 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13af503d1b68a9ec8649b1b1c0fa50fb#B#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:01,693 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/2aa2b360f36846839366f8c93e3405c5 is 50, key is test_row_0/B:col10/1731358441209/Put/seqid=0 2024-11-11T20:54:01,702 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13af503d1b68a9ec8649b1b1c0fa50fb#A#compaction#232 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:01,703 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/a98c5231204b41d88ae6b9fd80288e57 is 50, key is test_row_0/A:col10/1731358441209/Put/seqid=0 2024-11-11T20:54:01,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742099_1275 (size=12949) 2024-11-11T20:54:01,718 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/2aa2b360f36846839366f8c93e3405c5 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/2aa2b360f36846839366f8c93e3405c5 2024-11-11T20:54:01,726 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13af503d1b68a9ec8649b1b1c0fa50fb/B of 13af503d1b68a9ec8649b1b1c0fa50fb into 2aa2b360f36846839366f8c93e3405c5(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:01,726 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:54:01,726 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., storeName=13af503d1b68a9ec8649b1b1c0fa50fb/B, priority=13, startTime=1731358441679; duration=0sec 2024-11-11T20:54:01,726 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:54:01,726 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13af503d1b68a9ec8649b1b1c0fa50fb:B 2024-11-11T20:54:01,726 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:54:01,728 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:54:01,728 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 13af503d1b68a9ec8649b1b1c0fa50fb/C is initiating minor compaction (all files) 2024-11-11T20:54:01,728 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13af503d1b68a9ec8649b1b1c0fa50fb/C in TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:01,729 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/b6d96c3d39ef41589ff51629ddf9613b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/5e381986204042fdb257d87569f20a44, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/6c5cc821764b4c62a9b928f4153f8f86] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp, totalSize=36.6 K 2024-11-11T20:54:01,729 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting b6d96c3d39ef41589ff51629ddf9613b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=383, earliestPutTs=1731358439933 2024-11-11T20:54:01,729 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e381986204042fdb257d87569f20a44, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731358440021 2024-11-11T20:54:01,730 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c5cc821764b4c62a9b928f4153f8f86, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731358441201 2024-11-11T20:54:01,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742100_1276 (size=12949) 2024-11-11T20:54:01,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13af503d1b68a9ec8649b1b1c0fa50fb 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-11T20:54:01,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=A 2024-11-11T20:54:01,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T20:54:01,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=B 2024-11-11T20:54:01,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T20:54:01,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=C 2024-11-11T20:54:01,734 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T20:54:01,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:54:01,741 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/a98c5231204b41d88ae6b9fd80288e57 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/a98c5231204b41d88ae6b9fd80288e57 2024-11-11T20:54:01,747 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/6c4e2d0306ab4a6c976bca397ab1d46d is 50, key is test_row_0/A:col10/1731358441402/Put/seqid=0 2024-11-11T20:54:01,748 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13af503d1b68a9ec8649b1b1c0fa50fb/A of 13af503d1b68a9ec8649b1b1c0fa50fb into a98c5231204b41d88ae6b9fd80288e57(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:01,748 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:54:01,748 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., storeName=13af503d1b68a9ec8649b1b1c0fa50fb/A, priority=13, startTime=1731358441679; duration=0sec 2024-11-11T20:54:01,748 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13af503d1b68a9ec8649b1b1c0fa50fb#C#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:01,749 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:01,749 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13af503d1b68a9ec8649b1b1c0fa50fb:A 2024-11-11T20:54:01,749 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/b0a2c89648e14b47ab016c77ef582660 is 50, key is test_row_0/C:col10/1731358441209/Put/seqid=0 2024-11-11T20:54:01,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742102_1278 (size=12949) 2024-11-11T20:54:01,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742101_1277 (size=14741) 2024-11-11T20:54:01,761 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/b0a2c89648e14b47ab016c77ef582660 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/b0a2c89648e14b47ab016c77ef582660 2024-11-11T20:54:01,768 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 13af503d1b68a9ec8649b1b1c0fa50fb/C of 13af503d1b68a9ec8649b1b1c0fa50fb into b0a2c89648e14b47ab016c77ef582660(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:01,768 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:54:01,768 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., storeName=13af503d1b68a9ec8649b1b1c0fa50fb/C, priority=13, startTime=1731358441679; duration=0sec 2024-11-11T20:54:01,768 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:01,768 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13af503d1b68a9ec8649b1b1c0fa50fb:C 2024-11-11T20:54:01,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:01,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 222 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358501773, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:01,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:01,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:01,776 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:01,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T20:54:01,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:01,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:54:01,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:01,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:01,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:01,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:01,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 223 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358501885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:01,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:01,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:01,886 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:01,938 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T20:54:01,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:01,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:54:01,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:01,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:01,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:01,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:01,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T20:54:02,090 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T20:54:02,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:02,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:54:02,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:02,091 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:02,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:02,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:02,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:02,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 224 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358502096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:02,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:02,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:02,100 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:02,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/6c4e2d0306ab4a6c976bca397ab1d46d 2024-11-11T20:54:02,166 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/11aff73eb0c24f78a562b11d1f9da313 is 50, key is test_row_0/B:col10/1731358441402/Put/seqid=0 2024-11-11T20:54:02,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742103_1279 (size=12301) 2024-11-11T20:54:02,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T20:54:02,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:02,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:54:02,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:02,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:02,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:02,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:02,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T20:54:02,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:02,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:54:02,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:02,396 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:02,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:02,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:02,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:02,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 225 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358502405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:02,407 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:02,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:02,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:02,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T20:54:02,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T20:54:02,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:02,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:54:02,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:02,549 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:02,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:02,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:02,570 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/11aff73eb0c24f78a562b11d1f9da313 2024-11-11T20:54:02,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/1bd6689b208b4f55a6390af0a079c289 is 50, key is test_row_0/C:col10/1731358441402/Put/seqid=0 2024-11-11T20:54:02,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742104_1280 (size=12301) 2024-11-11T20:54:02,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T20:54:02,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:02,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:54:02,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:02,701 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:02,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:02,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:02,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T20:54:02,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:02,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:54:02,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:02,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:02,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:02,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:02,918 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 226 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358502916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:02,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:02,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:02,919 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:02,982 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=456 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/1bd6689b208b4f55a6390af0a079c289 2024-11-11T20:54:02,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/6c4e2d0306ab4a6c976bca397ab1d46d as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/6c4e2d0306ab4a6c976bca397ab1d46d 2024-11-11T20:54:02,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/6c4e2d0306ab4a6c976bca397ab1d46d, entries=200, sequenceid=456, filesize=14.4 K 2024-11-11T20:54:02,990 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/11aff73eb0c24f78a562b11d1f9da313 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/11aff73eb0c24f78a562b11d1f9da313 2024-11-11T20:54:02,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/11aff73eb0c24f78a562b11d1f9da313, entries=150, sequenceid=456, filesize=12.0 K 2024-11-11T20:54:02,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/1bd6689b208b4f55a6390af0a079c289 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/1bd6689b208b4f55a6390af0a079c289 2024-11-11T20:54:02,997 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/1bd6689b208b4f55a6390af0a079c289, entries=150, sequenceid=456, filesize=12.0 K 2024-11-11T20:54:02,998 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 13af503d1b68a9ec8649b1b1c0fa50fb in 1265ms, sequenceid=456, compaction requested=false 2024-11-11T20:54:02,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:54:03,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T20:54:03,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:03,007 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 13af503d1b68a9ec8649b1b1c0fa50fb 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-11T20:54:03,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=A 2024-11-11T20:54:03,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:03,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=B 2024-11-11T20:54:03,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:03,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=C 2024-11-11T20:54:03,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:03,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/63f7cacfb1c8484eab889e03d4439a44 is 50, key is test_row_0/A:col10/1731358441762/Put/seqid=0 2024-11-11T20:54:03,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742105_1281 (size=12301) 2024-11-11T20:54:03,416 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/63f7cacfb1c8484eab889e03d4439a44 2024-11-11T20:54:03,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/2e3eb1e8cf844a6da8746f6f3be49ae2 is 50, key is test_row_0/B:col10/1731358441762/Put/seqid=0 2024-11-11T20:54:03,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742106_1282 (size=12301) 2024-11-11T20:54:03,439 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/2e3eb1e8cf844a6da8746f6f3be49ae2 2024-11-11T20:54:03,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/d1096bb624254c82b2de9b22176f189a is 50, key is test_row_0/C:col10/1731358441762/Put/seqid=0 2024-11-11T20:54:03,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742107_1283 (size=12301) 2024-11-11T20:54:03,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T20:54:03,867 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=467 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/d1096bb624254c82b2de9b22176f189a 2024-11-11T20:54:03,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/63f7cacfb1c8484eab889e03d4439a44 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/63f7cacfb1c8484eab889e03d4439a44 2024-11-11T20:54:03,885 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/63f7cacfb1c8484eab889e03d4439a44, entries=150, sequenceid=467, filesize=12.0 K 2024-11-11T20:54:03,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/2e3eb1e8cf844a6da8746f6f3be49ae2 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/2e3eb1e8cf844a6da8746f6f3be49ae2 2024-11-11T20:54:03,892 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/2e3eb1e8cf844a6da8746f6f3be49ae2, entries=150, sequenceid=467, filesize=12.0 K 2024-11-11T20:54:03,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/d1096bb624254c82b2de9b22176f189a as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/d1096bb624254c82b2de9b22176f189a 2024-11-11T20:54:03,902 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/d1096bb624254c82b2de9b22176f189a, entries=150, sequenceid=467, filesize=12.0 K 2024-11-11T20:54:03,912 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=0 B/0 for 13af503d1b68a9ec8649b1b1c0fa50fb in 905ms, sequenceid=467, compaction requested=true 2024-11-11T20:54:03,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:54:03,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:03,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-11T20:54:03,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-11T20:54:03,917 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-11T20:54:03,917 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5860 sec 2024-11-11T20:54:03,921 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.5920 sec 2024-11-11T20:54:03,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:54:03,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13af503d1b68a9ec8649b1b1c0fa50fb 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T20:54:03,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=A 2024-11-11T20:54:03,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:03,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=B 2024-11-11T20:54:03,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:03,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=C 2024-11-11T20:54:03,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:03,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/53347335bce94bee90a2c0ab7ca140ab is 50, key is test_row_0/A:col10/1731358443955/Put/seqid=0 2024-11-11T20:54:03,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742108_1284 (size=12297) 2024-11-11T20:54:04,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358443982, max timestamp=1731358444053Num uniques 859; 2024-11-11T20:54:04,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358443982, max timestamp=1731358444053Num uniques 859; 2024-11-11T20:54:04,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:54:04,056 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:54:04,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358443982, max timestamp=1731358444053Num uniques 859; 2024-11-11T20:54:04,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:54:04,147 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:04,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 268 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358504144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:04,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:04,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:04,148 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:04,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:04,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 269 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358504255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:04,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:04,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:04,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:04,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:04,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41632 deadline: 1731358504296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:04,299 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in 13af503d1b68a9ec8649b1b1c0fa50fb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:54:04,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:04,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:04,299 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:04,310 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:04,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41666 deadline: 1731358504306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:04,311 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_0' in 13af503d1b68a9ec8649b1b1c0fa50fb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:54:04,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:04,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:04,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:04,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:04,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41662 deadline: 1731358504315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:04,318 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in 13af503d1b68a9ec8649b1b1c0fa50fb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:54:04,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:04,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:04,319 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:04,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:04,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41616 deadline: 1731358504335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:04,340 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in 13af503d1b68a9ec8649b1b1c0fa50fb of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8239 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:54:04,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:04,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:04,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:04,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/53347335bce94bee90a2c0ab7ca140ab 2024-11-11T20:54:04,373 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/2c61ee7c2eec45dc96fa3481a14d3da3 is 50, key is test_row_0/B:col10/1731358443955/Put/seqid=0 2024-11-11T20:54:04,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742109_1285 (size=9857) 2024-11-11T20:54:04,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:04,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 270 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358504466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:04,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:04,470 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:04,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/2c61ee7c2eec45dc96fa3481a14d3da3 2024-11-11T20:54:04,780 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:04,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 271 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358504776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:04,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:04,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:04,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:04,783 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/de194f361f2d4297b87fbb9f4de2ea38 is 50, key is test_row_0/C:col10/1731358443955/Put/seqid=0 2024-11-11T20:54:04,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742110_1286 (size=9857) 2024-11-11T20:54:05,188 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=477 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/de194f361f2d4297b87fbb9f4de2ea38 2024-11-11T20:54:05,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/53347335bce94bee90a2c0ab7ca140ab as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/53347335bce94bee90a2c0ab7ca140ab 2024-11-11T20:54:05,195 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/53347335bce94bee90a2c0ab7ca140ab, entries=150, sequenceid=477, filesize=12.0 K 2024-11-11T20:54:05,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/2c61ee7c2eec45dc96fa3481a14d3da3 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/2c61ee7c2eec45dc96fa3481a14d3da3 2024-11-11T20:54:05,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/2c61ee7c2eec45dc96fa3481a14d3da3, entries=100, sequenceid=477, filesize=9.6 K 2024-11-11T20:54:05,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/de194f361f2d4297b87fbb9f4de2ea38 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/de194f361f2d4297b87fbb9f4de2ea38 2024-11-11T20:54:05,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/de194f361f2d4297b87fbb9f4de2ea38, entries=100, sequenceid=477, filesize=9.6 K 2024-11-11T20:54:05,205 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 13af503d1b68a9ec8649b1b1c0fa50fb in 1247ms, sequenceid=477, compaction requested=true 2024-11-11T20:54:05,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:54:05,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13af503d1b68a9ec8649b1b1c0fa50fb:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:54:05,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:05,205 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:54:05,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13af503d1b68a9ec8649b1b1c0fa50fb:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:54:05,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:05,205 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:54:05,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 13af503d1b68a9ec8649b1b1c0fa50fb:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:54:05,205 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:54:05,206 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52288 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:54:05,206 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47408 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:54:05,206 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 13af503d1b68a9ec8649b1b1c0fa50fb/B is initiating minor compaction (all files) 2024-11-11T20:54:05,206 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 13af503d1b68a9ec8649b1b1c0fa50fb/A is initiating minor compaction (all files) 2024-11-11T20:54:05,206 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13af503d1b68a9ec8649b1b1c0fa50fb/A in TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:05,206 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13af503d1b68a9ec8649b1b1c0fa50fb/B in TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:05,206 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/2aa2b360f36846839366f8c93e3405c5, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/11aff73eb0c24f78a562b11d1f9da313, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/2e3eb1e8cf844a6da8746f6f3be49ae2, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/2c61ee7c2eec45dc96fa3481a14d3da3] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp, totalSize=46.3 K 2024-11-11T20:54:05,206 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/a98c5231204b41d88ae6b9fd80288e57, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/6c4e2d0306ab4a6c976bca397ab1d46d, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/63f7cacfb1c8484eab889e03d4439a44, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/53347335bce94bee90a2c0ab7ca140ab] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp, totalSize=51.1 K 2024-11-11T20:54:05,207 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting a98c5231204b41d88ae6b9fd80288e57, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731358441201 2024-11-11T20:54:05,207 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 2aa2b360f36846839366f8c93e3405c5, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731358441201 2024-11-11T20:54:05,207 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c4e2d0306ab4a6c976bca397ab1d46d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731358441372 2024-11-11T20:54:05,207 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 11aff73eb0c24f78a562b11d1f9da313, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731358441372 2024-11-11T20:54:05,207 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63f7cacfb1c8484eab889e03d4439a44, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1731358441762 2024-11-11T20:54:05,207 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e3eb1e8cf844a6da8746f6f3be49ae2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1731358441762 2024-11-11T20:54:05,208 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 53347335bce94bee90a2c0ab7ca140ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1731358443949 2024-11-11T20:54:05,208 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c61ee7c2eec45dc96fa3481a14d3da3, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1731358443949 2024-11-11T20:54:05,216 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13af503d1b68a9ec8649b1b1c0fa50fb#B#compaction#243 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:05,216 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/79c2a4a362c6405888a494c08e4e01bc is 50, key is test_row_0/B:col10/1731358443955/Put/seqid=0 2024-11-11T20:54:05,220 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13af503d1b68a9ec8649b1b1c0fa50fb#A#compaction#244 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:05,220 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/919bc25c242546afa70c8c2c19908e33 is 50, key is test_row_0/A:col10/1731358443955/Put/seqid=0 2024-11-11T20:54:05,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742111_1287 (size=13085) 2024-11-11T20:54:05,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742112_1288 (size=13085) 2024-11-11T20:54:05,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:54:05,299 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 13af503d1b68a9ec8649b1b1c0fa50fb 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-11T20:54:05,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=A 2024-11-11T20:54:05,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T20:54:05,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=B 2024-11-11T20:54:05,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T20:54:05,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=C 2024-11-11T20:54:05,299 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T20:54:05,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/b45c2d122f4e4d1192bf262b39a7f101 is 50, key is test_row_0/A:col10/1731358445297/Put/seqid=0 2024-11-11T20:54:05,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742113_1289 (size=14741) 2024-11-11T20:54:05,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 279 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358505330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:05,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:05,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:05,334 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:05,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:05,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358505446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:05,448 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:05,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T20:54:05,486 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:54:05,486 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:54:05,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-11T20:54:05,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T20:54:05,488 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:54:05,488 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:54:05,488 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:54:05,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T20:54:05,634 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/79c2a4a362c6405888a494c08e4e01bc as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/79c2a4a362c6405888a494c08e4e01bc 2024-11-11T20:54:05,636 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/919bc25c242546afa70c8c2c19908e33 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/919bc25c242546afa70c8c2c19908e33 2024-11-11T20:54:05,640 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T20:54:05,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:05,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:54:05,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:05,641 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:05,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:05,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:05,643 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13af503d1b68a9ec8649b1b1c0fa50fb/B of 13af503d1b68a9ec8649b1b1c0fa50fb into 79c2a4a362c6405888a494c08e4e01bc(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:05,643 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:54:05,643 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., storeName=13af503d1b68a9ec8649b1b1c0fa50fb/B, priority=12, startTime=1731358445205; duration=0sec 2024-11-11T20:54:05,644 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:54:05,644 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13af503d1b68a9ec8649b1b1c0fa50fb:B 2024-11-11T20:54:05,644 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:54:05,645 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 47408 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:54:05,645 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13af503d1b68a9ec8649b1b1c0fa50fb/A of 13af503d1b68a9ec8649b1b1c0fa50fb into 919bc25c242546afa70c8c2c19908e33(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:05,645 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 13af503d1b68a9ec8649b1b1c0fa50fb/C is initiating minor compaction (all files) 2024-11-11T20:54:05,645 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:54:05,645 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., storeName=13af503d1b68a9ec8649b1b1c0fa50fb/A, priority=12, startTime=1731358445205; duration=0sec 2024-11-11T20:54:05,645 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 13af503d1b68a9ec8649b1b1c0fa50fb/C in TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:05,645 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:05,645 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13af503d1b68a9ec8649b1b1c0fa50fb:A 2024-11-11T20:54:05,645 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/b0a2c89648e14b47ab016c77ef582660, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/1bd6689b208b4f55a6390af0a079c289, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/d1096bb624254c82b2de9b22176f189a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/de194f361f2d4297b87fbb9f4de2ea38] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp, totalSize=46.3 K 2024-11-11T20:54:05,646 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting b0a2c89648e14b47ab016c77ef582660, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731358441201 2024-11-11T20:54:05,646 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bd6689b208b4f55a6390af0a079c289, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=456, earliestPutTs=1731358441372 2024-11-11T20:54:05,646 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting d1096bb624254c82b2de9b22176f189a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=467, earliestPutTs=1731358441762 2024-11-11T20:54:05,646 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting de194f361f2d4297b87fbb9f4de2ea38, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=477, earliestPutTs=1731358443949 2024-11-11T20:54:05,652 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 13af503d1b68a9ec8649b1b1c0fa50fb#C#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:05,652 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/19089399546944e5a78302a81fceea88 is 50, key is test_row_0/C:col10/1731358443955/Put/seqid=0 2024-11-11T20:54:05,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742114_1290 (size=13085) 2024-11-11T20:54:05,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358505656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:05,658 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:05,708 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/b45c2d122f4e4d1192bf262b39a7f101 2024-11-11T20:54:05,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/004bb318ec044fdcbbad4dad41601595 is 50, key is test_row_0/B:col10/1731358445297/Put/seqid=0 2024-11-11T20:54:05,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742115_1291 (size=12301) 2024-11-11T20:54:05,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T20:54:05,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:05,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:54:05,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:05,794 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:05,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:05,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:05,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T20:54:05,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T20:54:05,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:05,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:54:05,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:05,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:05,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:05,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:05,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:05,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358505965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:05,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:05,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:05,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:06,060 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/19089399546944e5a78302a81fceea88 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/19089399546944e5a78302a81fceea88 2024-11-11T20:54:06,064 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 13af503d1b68a9ec8649b1b1c0fa50fb/C of 13af503d1b68a9ec8649b1b1c0fa50fb into 19089399546944e5a78302a81fceea88(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:06,064 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:54:06,064 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., storeName=13af503d1b68a9ec8649b1b1c0fa50fb/C, priority=12, startTime=1731358445205; duration=0sec 2024-11-11T20:54:06,064 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:06,064 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 13af503d1b68a9ec8649b1b1c0fa50fb:C 2024-11-11T20:54:06,099 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T20:54:06,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:06,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:54:06,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:06,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:06,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:06,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:06,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T20:54:06,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/004bb318ec044fdcbbad4dad41601595 2024-11-11T20:54:06,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/9483005fda5140179408f1ee10f10bed is 50, key is test_row_0/C:col10/1731358445297/Put/seqid=0 2024-11-11T20:54:06,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742116_1292 (size=12301) 2024-11-11T20:54:06,252 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T20:54:06,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:06,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:54:06,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:06,253 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:06,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:06,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:06,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T20:54:06,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:06,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:54:06,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:06,407 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:06,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:06,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:06,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41592 deadline: 1731358506475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:06,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:06,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:06,479 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:06,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=515 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/9483005fda5140179408f1ee10f10bed 2024-11-11T20:54:06,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/b45c2d122f4e4d1192bf262b39a7f101 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/b45c2d122f4e4d1192bf262b39a7f101 2024-11-11T20:54:06,538 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/b45c2d122f4e4d1192bf262b39a7f101, entries=200, sequenceid=515, filesize=14.4 K 2024-11-11T20:54:06,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/004bb318ec044fdcbbad4dad41601595 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/004bb318ec044fdcbbad4dad41601595 2024-11-11T20:54:06,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/004bb318ec044fdcbbad4dad41601595, entries=150, sequenceid=515, filesize=12.0 K 2024-11-11T20:54:06,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/9483005fda5140179408f1ee10f10bed as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/9483005fda5140179408f1ee10f10bed 2024-11-11T20:54:06,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/9483005fda5140179408f1ee10f10bed, entries=150, sequenceid=515, filesize=12.0 K 2024-11-11T20:54:06,549 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~405.56 KB/415296, currentSize=40.25 KB/41220 for 13af503d1b68a9ec8649b1b1c0fa50fb in 1250ms, sequenceid=515, compaction requested=false 2024-11-11T20:54:06,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:54:06,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T20:54:06,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:06,560 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 13af503d1b68a9ec8649b1b1c0fa50fb 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-11T20:54:06,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=A 2024-11-11T20:54:06,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:06,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=B 2024-11-11T20:54:06,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:06,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=C 2024-11-11T20:54:06,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:06,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/dd5120df2dbd4c779ff5067bbc13aba2 is 50, key is test_row_0/A:col10/1731358445328/Put/seqid=0 2024-11-11T20:54:06,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742117_1293 (size=12301) 2024-11-11T20:54:06,620 INFO [Thread-1039 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1039. 2024-11-11T20:54:06,621 DEBUG [Thread-1039 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:54:06,621 DEBUG [Thread-1039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:06,621 DEBUG [Thread-1039 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:06,621 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:54:06,621 INFO [Thread-1037 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1037. 2024-11-11T20:54:06,621 DEBUG [Thread-1037 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:54:06,621 DEBUG [Thread-1037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:06,621 DEBUG [Thread-1037 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:06,621 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:54:06,621 INFO [Thread-1043 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1043. 2024-11-11T20:54:06,621 DEBUG [Thread-1043 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:54:06,622 DEBUG [Thread-1043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:06,622 DEBUG [Thread-1043 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:06,622 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:54:06,622 INFO [Thread-1045 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1045. 2024-11-11T20:54:06,622 DEBUG [Thread-1045 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:54:06,622 DEBUG [Thread-1045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:06,622 DEBUG [Thread-1045 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:06,622 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:54:06,623 INFO [Thread-1041 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1041. 2024-11-11T20:54:06,623 DEBUG [Thread-1041 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:54:06,623 DEBUG [Thread-1041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:06,623 DEBUG [Thread-1041 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:06,623 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:54:06,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T20:54:06,969 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/dd5120df2dbd4c779ff5067bbc13aba2 2024-11-11T20:54:06,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/e38c9cd3c548457d9c8ce0ce2ec41fc2 is 50, key is test_row_0/B:col10/1731358445328/Put/seqid=0 2024-11-11T20:54:06,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742118_1294 (size=12301) 2024-11-11T20:54:07,380 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/e38c9cd3c548457d9c8ce0ce2ec41fc2 2024-11-11T20:54:07,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/5cdfc34f69804d43b4bcb6644b590cd2 is 50, key is test_row_0/C:col10/1731358445328/Put/seqid=0 2024-11-11T20:54:07,399 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742119_1295 (size=12301) 2024-11-11T20:54:07,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:54:07,488 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. as already flushing 2024-11-11T20:54:07,488 INFO [Thread-1032 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1032. 2024-11-11T20:54:07,488 DEBUG [Thread-1032 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:54:07,488 DEBUG [Thread-1032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:07,488 DEBUG [Thread-1032 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:07,488 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:54:07,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T20:54:07,800 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/5cdfc34f69804d43b4bcb6644b590cd2 2024-11-11T20:54:07,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/dd5120df2dbd4c779ff5067bbc13aba2 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/dd5120df2dbd4c779ff5067bbc13aba2 2024-11-11T20:54:07,815 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/dd5120df2dbd4c779ff5067bbc13aba2, entries=150, sequenceid=527, filesize=12.0 K 2024-11-11T20:54:07,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/e38c9cd3c548457d9c8ce0ce2ec41fc2 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/e38c9cd3c548457d9c8ce0ce2ec41fc2 2024-11-11T20:54:07,819 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/e38c9cd3c548457d9c8ce0ce2ec41fc2, entries=150, sequenceid=527, filesize=12.0 K 2024-11-11T20:54:07,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/5cdfc34f69804d43b4bcb6644b590cd2 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/5cdfc34f69804d43b4bcb6644b590cd2 2024-11-11T20:54:07,824 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/5cdfc34f69804d43b4bcb6644b590cd2, entries=150, sequenceid=527, filesize=12.0 K 2024-11-11T20:54:07,825 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=6.71 KB/6870 for 13af503d1b68a9ec8649b1b1c0fa50fb in 1265ms, sequenceid=527, compaction requested=true 2024-11-11T20:54:07,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 13af503d1b68a9ec8649b1b1c0fa50fb: 2024-11-11T20:54:07,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:07,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-11T20:54:07,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-11T20:54:07,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-11T20:54:07,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3380 sec 2024-11-11T20:54:07,830 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 2.3430 sec 2024-11-11T20:54:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T20:54:09,657 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:54:10,577 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T20:54:14,350 INFO [Thread-1034 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1034. 2024-11-11T20:54:14,350 DEBUG [Thread-1034 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:54:14,350 DEBUG [Thread-1034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:14,350 DEBUG [Thread-1034 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:14,350 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:54:14,379 INFO [Thread-1026 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1026. 2024-11-11T20:54:14,380 DEBUG [Thread-1026 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:54:14,380 DEBUG [Thread-1026 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:14,380 DEBUG [Thread-1026 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:14,380 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:54:14,389 INFO [Thread-1028 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1028. 2024-11-11T20:54:14,389 DEBUG [Thread-1028 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:54:14,389 DEBUG [Thread-1028 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:14,389 DEBUG [Thread-1028 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:14,389 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:54:14,399 INFO [Thread-1030 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1030. 2024-11-11T20:54:14,399 DEBUG [Thread-1030 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:54:14,399 DEBUG [Thread-1030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:14,399 DEBUG [Thread-1030 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:14,399 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:54:14,400 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T20:54:14,400 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-11T20:54:14,400 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 93 2024-11-11T20:54:14,400 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 17 2024-11-11T20:54:14,400 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 244 2024-11-11T20:54:14,400 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-11T20:54:14,400 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T20:54:14,400 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3386 2024-11-11T20:54:14,400 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3515 2024-11-11T20:54:14,400 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3185 2024-11-11T20:54:14,400 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3397 2024-11-11T20:54:14,400 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3506 2024-11-11T20:54:14,400 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T20:54:14,400 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T20:54:14,401 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:54:14,401 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:14,401 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:14,401 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:54:14,402 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T20:54:14,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T20:54:14,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T20:54:14,408 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358454407"}]},"ts":"1731358454407"} 2024-11-11T20:54:14,409 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T20:54:14,410 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T20:54:14,410 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T20:54:14,412 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13af503d1b68a9ec8649b1b1c0fa50fb, UNASSIGN}] 2024-11-11T20:54:14,413 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13af503d1b68a9ec8649b1b1c0fa50fb, UNASSIGN 2024-11-11T20:54:14,414 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=13af503d1b68a9ec8649b1b1c0fa50fb, regionState=CLOSING, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:14,416 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=13af503d1b68a9ec8649b1b1c0fa50fb, UNASSIGN because future has completed 2024-11-11T20:54:14,416 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T20:54:14,416 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; CloseRegionProcedure 13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066}] 2024-11-11T20:54:14,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T20:54:14,570 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(122): Close 13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:54:14,570 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T20:54:14,570 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1722): Closing 13af503d1b68a9ec8649b1b1c0fa50fb, disabling compactions & flushes 2024-11-11T20:54:14,570 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:14,570 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:14,570 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. after waiting 0 ms 2024-11-11T20:54:14,570 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:14,570 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(2902): Flushing 13af503d1b68a9ec8649b1b1c0fa50fb 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T20:54:14,571 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=A 2024-11-11T20:54:14,571 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:14,571 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=B 2024-11-11T20:54:14,571 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:14,571 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 13af503d1b68a9ec8649b1b1c0fa50fb, store=C 2024-11-11T20:54:14,571 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:14,581 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/0ad9f7ccb0724d2280f8fe58eaa856a0 is 50, key is test_row_0/A:col10/1731358454396/Put/seqid=0 2024-11-11T20:54:14,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742120_1296 (size=9857) 2024-11-11T20:54:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T20:54:14,987 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/0ad9f7ccb0724d2280f8fe58eaa856a0 2024-11-11T20:54:15,002 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/d9cea52b808c4362bd48e8e06401ee31 is 50, key is test_row_0/B:col10/1731358454396/Put/seqid=0 2024-11-11T20:54:15,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742121_1297 (size=9857) 2024-11-11T20:54:15,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T20:54:15,409 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/d9cea52b808c4362bd48e8e06401ee31 2024-11-11T20:54:15,423 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/c53cae5fea694d5eb61f37e9a197aada is 50, key is test_row_0/C:col10/1731358454396/Put/seqid=0 2024-11-11T20:54:15,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742122_1298 (size=9857) 2024-11-11T20:54:15,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T20:54:15,828 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=535 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/c53cae5fea694d5eb61f37e9a197aada 2024-11-11T20:54:15,833 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/A/0ad9f7ccb0724d2280f8fe58eaa856a0 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/0ad9f7ccb0724d2280f8fe58eaa856a0 2024-11-11T20:54:15,837 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/0ad9f7ccb0724d2280f8fe58eaa856a0, entries=100, sequenceid=535, filesize=9.6 K 2024-11-11T20:54:15,838 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/B/d9cea52b808c4362bd48e8e06401ee31 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/d9cea52b808c4362bd48e8e06401ee31 2024-11-11T20:54:15,842 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/d9cea52b808c4362bd48e8e06401ee31, entries=100, sequenceid=535, filesize=9.6 K 2024-11-11T20:54:15,843 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/.tmp/C/c53cae5fea694d5eb61f37e9a197aada as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/c53cae5fea694d5eb61f37e9a197aada 2024-11-11T20:54:15,846 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/c53cae5fea694d5eb61f37e9a197aada, entries=100, sequenceid=535, filesize=9.6 K 2024-11-11T20:54:15,847 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 13af503d1b68a9ec8649b1b1c0fa50fb in 1276ms, sequenceid=535, compaction requested=true 2024-11-11T20:54:15,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/b6f4e26e5cab4c128950de064ceccdad, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/e419c0ba9e044fca9e944b778ba0caea, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/3207e2065acc413d92693adefe400d95, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/f9d10c6348734859a98e2030b0493db1, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/92abd2ed15be4bc8af71db574d1eda8c, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/f465d0ba9a3e47ce96b32eb640ce5a1d, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/458bedb7ac984de59ca5ace38d3126f7, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/9a3edb284f9140c18c8af43406a630b9, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/91591f107067456d82dbdcfabde943ad, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/5c18e2bee0444267ad3a9731832cefc5, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/6aab9c39fadf42d4bad4d64d99eaf1f0, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/cc78b1f15b854d0486106cdb79da5998, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/f58c102b3fa94290a3909af3e200c504, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/123fd5f1ac62453f9f73b9c435d9ec0b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/edd313e0c1034f90bd6671847481ac9b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/bf70373a80d341248e85c661dd9a4c4a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/d24044560e5240eb8a84c3d3d7f75c10, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/a4d739a19cb741239e46f2f384429c7e, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/15e27f6d871b401496fa168e3ae09cf8, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/a98c5231204b41d88ae6b9fd80288e57, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/6c4e2d0306ab4a6c976bca397ab1d46d, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/63f7cacfb1c8484eab889e03d4439a44, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/53347335bce94bee90a2c0ab7ca140ab] to archive 2024-11-11T20:54:15,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:54:15,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/b6f4e26e5cab4c128950de064ceccdad to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/b6f4e26e5cab4c128950de064ceccdad 2024-11-11T20:54:15,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/e419c0ba9e044fca9e944b778ba0caea to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/e419c0ba9e044fca9e944b778ba0caea 2024-11-11T20:54:15,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/3207e2065acc413d92693adefe400d95 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/3207e2065acc413d92693adefe400d95 2024-11-11T20:54:15,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/f9d10c6348734859a98e2030b0493db1 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/f9d10c6348734859a98e2030b0493db1 2024-11-11T20:54:15,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/92abd2ed15be4bc8af71db574d1eda8c to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/92abd2ed15be4bc8af71db574d1eda8c 2024-11-11T20:54:15,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/f465d0ba9a3e47ce96b32eb640ce5a1d to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/f465d0ba9a3e47ce96b32eb640ce5a1d 2024-11-11T20:54:15,854 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/458bedb7ac984de59ca5ace38d3126f7 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/458bedb7ac984de59ca5ace38d3126f7 2024-11-11T20:54:15,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/9a3edb284f9140c18c8af43406a630b9 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/9a3edb284f9140c18c8af43406a630b9 2024-11-11T20:54:15,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/91591f107067456d82dbdcfabde943ad to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/91591f107067456d82dbdcfabde943ad 2024-11-11T20:54:15,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/5c18e2bee0444267ad3a9731832cefc5 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/5c18e2bee0444267ad3a9731832cefc5 2024-11-11T20:54:15,858 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/6aab9c39fadf42d4bad4d64d99eaf1f0 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/6aab9c39fadf42d4bad4d64d99eaf1f0 2024-11-11T20:54:15,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/cc78b1f15b854d0486106cdb79da5998 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/cc78b1f15b854d0486106cdb79da5998 2024-11-11T20:54:15,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/f58c102b3fa94290a3909af3e200c504 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/f58c102b3fa94290a3909af3e200c504 2024-11-11T20:54:15,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/123fd5f1ac62453f9f73b9c435d9ec0b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/123fd5f1ac62453f9f73b9c435d9ec0b 2024-11-11T20:54:15,861 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/edd313e0c1034f90bd6671847481ac9b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/edd313e0c1034f90bd6671847481ac9b 2024-11-11T20:54:15,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/bf70373a80d341248e85c661dd9a4c4a to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/bf70373a80d341248e85c661dd9a4c4a 2024-11-11T20:54:15,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/d24044560e5240eb8a84c3d3d7f75c10 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/d24044560e5240eb8a84c3d3d7f75c10 2024-11-11T20:54:15,863 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/a4d739a19cb741239e46f2f384429c7e to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/a4d739a19cb741239e46f2f384429c7e 2024-11-11T20:54:15,864 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/15e27f6d871b401496fa168e3ae09cf8 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/15e27f6d871b401496fa168e3ae09cf8 2024-11-11T20:54:15,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/a98c5231204b41d88ae6b9fd80288e57 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/a98c5231204b41d88ae6b9fd80288e57 2024-11-11T20:54:15,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/6c4e2d0306ab4a6c976bca397ab1d46d to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/6c4e2d0306ab4a6c976bca397ab1d46d 2024-11-11T20:54:15,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/63f7cacfb1c8484eab889e03d4439a44 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/63f7cacfb1c8484eab889e03d4439a44 2024-11-11T20:54:15,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/53347335bce94bee90a2c0ab7ca140ab to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/53347335bce94bee90a2c0ab7ca140ab 2024-11-11T20:54:15,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/35343a749f114ec9ad4f35a7d72b8e7f, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/1e71ef14678645d1bc2ee0bd409a26d7, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/539f91f460914443bdd320c98cbc3649, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/b918f482991d4496b79488306eba2918, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/2fd927834087482b9cc3c0975a8c61c1, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/5c6cced84bd548ccbdc2837a460e19c3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/17efaaa12a7448b1b708a9099176e595, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/808f7ff9e7da4d1989ed17ec4ab246c3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/4f951d3af28249c5a1ef77412aa81373, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/af87063d56fa49dcbb5a905f89f40154, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/384d2c4d043649cc9793562aa2ec5b20, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/b93f3a3176ee4f0b803a2821f2549124, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/c64940184bc24d7da56100d603008a1d, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/bf0db16c82364ef397bf19e40fe45e75, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/86e89264ac63482abd3f4d02c415bfc8, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/7eda733f92a946c2b423fb90433da258, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/14790f6ff4344b209fd3e06e60cce7f3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/e56818ee2f76497d9606def4020508d9, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/2aa2b360f36846839366f8c93e3405c5, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/54e45cf14a944046a9f5f3e498fc8e72, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/11aff73eb0c24f78a562b11d1f9da313, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/2e3eb1e8cf844a6da8746f6f3be49ae2, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/2c61ee7c2eec45dc96fa3481a14d3da3] to archive 2024-11-11T20:54:15,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:54:15,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/35343a749f114ec9ad4f35a7d72b8e7f to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/35343a749f114ec9ad4f35a7d72b8e7f 2024-11-11T20:54:15,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/1e71ef14678645d1bc2ee0bd409a26d7 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/1e71ef14678645d1bc2ee0bd409a26d7 2024-11-11T20:54:15,872 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/539f91f460914443bdd320c98cbc3649 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/539f91f460914443bdd320c98cbc3649 2024-11-11T20:54:15,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/b918f482991d4496b79488306eba2918 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/b918f482991d4496b79488306eba2918 2024-11-11T20:54:15,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/2fd927834087482b9cc3c0975a8c61c1 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/2fd927834087482b9cc3c0975a8c61c1 2024-11-11T20:54:15,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/5c6cced84bd548ccbdc2837a460e19c3 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/5c6cced84bd548ccbdc2837a460e19c3 2024-11-11T20:54:15,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/17efaaa12a7448b1b708a9099176e595 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/17efaaa12a7448b1b708a9099176e595 2024-11-11T20:54:15,876 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/808f7ff9e7da4d1989ed17ec4ab246c3 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/808f7ff9e7da4d1989ed17ec4ab246c3 2024-11-11T20:54:15,877 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/4f951d3af28249c5a1ef77412aa81373 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/4f951d3af28249c5a1ef77412aa81373 2024-11-11T20:54:15,878 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/af87063d56fa49dcbb5a905f89f40154 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/af87063d56fa49dcbb5a905f89f40154 2024-11-11T20:54:15,879 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/384d2c4d043649cc9793562aa2ec5b20 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/384d2c4d043649cc9793562aa2ec5b20 2024-11-11T20:54:15,880 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/b93f3a3176ee4f0b803a2821f2549124 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/b93f3a3176ee4f0b803a2821f2549124 2024-11-11T20:54:15,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/c64940184bc24d7da56100d603008a1d to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/c64940184bc24d7da56100d603008a1d 2024-11-11T20:54:15,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/bf0db16c82364ef397bf19e40fe45e75 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/bf0db16c82364ef397bf19e40fe45e75 2024-11-11T20:54:15,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/86e89264ac63482abd3f4d02c415bfc8 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/86e89264ac63482abd3f4d02c415bfc8 2024-11-11T20:54:15,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/7eda733f92a946c2b423fb90433da258 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/7eda733f92a946c2b423fb90433da258 2024-11-11T20:54:15,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/14790f6ff4344b209fd3e06e60cce7f3 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/14790f6ff4344b209fd3e06e60cce7f3 2024-11-11T20:54:15,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/e56818ee2f76497d9606def4020508d9 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/e56818ee2f76497d9606def4020508d9 2024-11-11T20:54:15,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/2aa2b360f36846839366f8c93e3405c5 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/2aa2b360f36846839366f8c93e3405c5 2024-11-11T20:54:15,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/54e45cf14a944046a9f5f3e498fc8e72 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/54e45cf14a944046a9f5f3e498fc8e72 2024-11-11T20:54:15,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/11aff73eb0c24f78a562b11d1f9da313 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/11aff73eb0c24f78a562b11d1f9da313 2024-11-11T20:54:15,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/2e3eb1e8cf844a6da8746f6f3be49ae2 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/2e3eb1e8cf844a6da8746f6f3be49ae2 2024-11-11T20:54:15,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/2c61ee7c2eec45dc96fa3481a14d3da3 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/2c61ee7c2eec45dc96fa3481a14d3da3 2024-11-11T20:54:15,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/13cd8c1afb214ba495a0e16eac6f3e1f, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/8bdc12c85b054e26981d0861354ea4a4, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/8e1ac923904543aea28c168fb446acd4, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/3be8eb6ee08d40a398646a7f9e937791, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/63fe0b2a74d44ad0a56cb6ef46e2ce88, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/00ae9a51a4134e6e885ac5b4b933f925, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/6c1da9bb828f45809b1fc12ccd49e206, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/43a27c79e08a4a30bc7841bd08f27965, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/0985038e6ffe470fb081cb09424bb5e9, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/1d78e9f86890445bbaf793417d9b625a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/22929d862d854cbd9006100f1e0819f0, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/46d60b744bd4444498108f3ce58e1133, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/cbad3b14db944c32a3cb2984ea3df010, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/910b887acc9d4d529350cdbff65b3f27, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/965232c7188a43c4aec92f3ab4d088ee, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/b6d96c3d39ef41589ff51629ddf9613b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/7ec7e7b8999a4f5eaa2b8083c6970965, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/5e381986204042fdb257d87569f20a44, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/b0a2c89648e14b47ab016c77ef582660, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/6c5cc821764b4c62a9b928f4153f8f86, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/1bd6689b208b4f55a6390af0a079c289, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/d1096bb624254c82b2de9b22176f189a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/de194f361f2d4297b87fbb9f4de2ea38] to archive 2024-11-11T20:54:15,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:54:15,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/13cd8c1afb214ba495a0e16eac6f3e1f to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/13cd8c1afb214ba495a0e16eac6f3e1f 2024-11-11T20:54:15,894 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/8bdc12c85b054e26981d0861354ea4a4 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/8bdc12c85b054e26981d0861354ea4a4 2024-11-11T20:54:15,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/8e1ac923904543aea28c168fb446acd4 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/8e1ac923904543aea28c168fb446acd4 2024-11-11T20:54:15,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/3be8eb6ee08d40a398646a7f9e937791 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/3be8eb6ee08d40a398646a7f9e937791 2024-11-11T20:54:15,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/63fe0b2a74d44ad0a56cb6ef46e2ce88 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/63fe0b2a74d44ad0a56cb6ef46e2ce88 2024-11-11T20:54:15,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/00ae9a51a4134e6e885ac5b4b933f925 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/00ae9a51a4134e6e885ac5b4b933f925 2024-11-11T20:54:15,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/6c1da9bb828f45809b1fc12ccd49e206 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/6c1da9bb828f45809b1fc12ccd49e206 2024-11-11T20:54:15,899 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/43a27c79e08a4a30bc7841bd08f27965 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/43a27c79e08a4a30bc7841bd08f27965 2024-11-11T20:54:15,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/0985038e6ffe470fb081cb09424bb5e9 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/0985038e6ffe470fb081cb09424bb5e9 2024-11-11T20:54:15,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/1d78e9f86890445bbaf793417d9b625a to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/1d78e9f86890445bbaf793417d9b625a 2024-11-11T20:54:15,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/22929d862d854cbd9006100f1e0819f0 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/22929d862d854cbd9006100f1e0819f0 2024-11-11T20:54:15,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/46d60b744bd4444498108f3ce58e1133 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/46d60b744bd4444498108f3ce58e1133 2024-11-11T20:54:15,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/cbad3b14db944c32a3cb2984ea3df010 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/cbad3b14db944c32a3cb2984ea3df010 2024-11-11T20:54:15,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/910b887acc9d4d529350cdbff65b3f27 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/910b887acc9d4d529350cdbff65b3f27 2024-11-11T20:54:15,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/965232c7188a43c4aec92f3ab4d088ee to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/965232c7188a43c4aec92f3ab4d088ee 2024-11-11T20:54:15,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/b6d96c3d39ef41589ff51629ddf9613b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/b6d96c3d39ef41589ff51629ddf9613b 2024-11-11T20:54:15,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/7ec7e7b8999a4f5eaa2b8083c6970965 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/7ec7e7b8999a4f5eaa2b8083c6970965 2024-11-11T20:54:15,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/5e381986204042fdb257d87569f20a44 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/5e381986204042fdb257d87569f20a44 2024-11-11T20:54:15,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/b0a2c89648e14b47ab016c77ef582660 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/b0a2c89648e14b47ab016c77ef582660 2024-11-11T20:54:15,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/6c5cc821764b4c62a9b928f4153f8f86 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/6c5cc821764b4c62a9b928f4153f8f86 2024-11-11T20:54:15,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/1bd6689b208b4f55a6390af0a079c289 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/1bd6689b208b4f55a6390af0a079c289 2024-11-11T20:54:15,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/d1096bb624254c82b2de9b22176f189a to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/d1096bb624254c82b2de9b22176f189a 2024-11-11T20:54:15,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/de194f361f2d4297b87fbb9f4de2ea38 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/de194f361f2d4297b87fbb9f4de2ea38 2024-11-11T20:54:15,913 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/recovered.edits/538.seqid, newMaxSeqId=538, maxSeqId=1 2024-11-11T20:54:15,913 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. 2024-11-11T20:54:15,913 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] regionserver.HRegion(1676): Region close journal for 13af503d1b68a9ec8649b1b1c0fa50fb: Waiting for close lock at 1731358454570Running coprocessor pre-close hooks at 1731358454570Disabling compacts and flushes for region at 1731358454570Disabling writes for close at 1731358454570Obtaining lock to block concurrent updates at 1731358454570Preparing flush snapshotting stores in 13af503d1b68a9ec8649b1b1c0fa50fb at 1731358454570Finished memstore snapshotting TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731358454572 (+2 ms)Flushing stores of TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb. at 1731358454573 (+1 ms)Flushing 13af503d1b68a9ec8649b1b1c0fa50fb/A: creating writer at 1731358454573Flushing 13af503d1b68a9ec8649b1b1c0fa50fb/A: appending metadata at 1731358454580 (+7 ms)Flushing 13af503d1b68a9ec8649b1b1c0fa50fb/A: closing flushed file at 1731358454580Flushing 13af503d1b68a9ec8649b1b1c0fa50fb/B: creating writer at 1731358454998 (+418 ms)Flushing 13af503d1b68a9ec8649b1b1c0fa50fb/B: appending metadata at 1731358455002 (+4 ms)Flushing 13af503d1b68a9ec8649b1b1c0fa50fb/B: closing flushed file at 1731358455002Flushing 13af503d1b68a9ec8649b1b1c0fa50fb/C: creating writer at 1731358455418 (+416 ms)Flushing 13af503d1b68a9ec8649b1b1c0fa50fb/C: appending metadata at 1731358455422 (+4 ms)Flushing 13af503d1b68a9ec8649b1b1c0fa50fb/C: closing flushed file at 1731358455422Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3403d115: reopening flushed file at 1731358455832 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e0a5cd6: reopening flushed file at 1731358455837 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5bc0ded6: reopening flushed file at 1731358455842 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 13af503d1b68a9ec8649b1b1c0fa50fb in 1276ms, sequenceid=535, compaction requested=true at 1731358455847 (+5 ms)Writing region close event to WAL at 1731358455910 (+63 ms)Running coprocessor post-close hooks at 1731358455913 (+3 ms)Closed at 1731358455913 2024-11-11T20:54:15,915 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=75}] handler.UnassignRegionHandler(157): Closed 13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:54:15,915 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=13af503d1b68a9ec8649b1b1c0fa50fb, regionState=CLOSED 2024-11-11T20:54:15,916 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=75, ppid=74, state=RUNNABLE, hasLock=false; CloseRegionProcedure 13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 because future has completed 2024-11-11T20:54:15,918 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-11T20:54:15,918 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; CloseRegionProcedure 13af503d1b68a9ec8649b1b1c0fa50fb, server=51ca66f7ee3c,44669,1731358362066 in 1.5010 sec 2024-11-11T20:54:15,920 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=73 2024-11-11T20:54:15,920 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=73, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=13af503d1b68a9ec8649b1b1c0fa50fb, UNASSIGN in 1.5060 sec 2024-11-11T20:54:15,921 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-11T20:54:15,921 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5100 sec 2024-11-11T20:54:15,922 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358455922"}]},"ts":"1731358455922"} 2024-11-11T20:54:15,923 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T20:54:15,923 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T20:54:15,924 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5210 sec 2024-11-11T20:54:16,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T20:54:16,557 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:54:16,558 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T20:54:16,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:54:16,562 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=76, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:54:16,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T20:54:16,563 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=76, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:54:16,566 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:54:16,570 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A, FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B, FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C, FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/recovered.edits] 2024-11-11T20:54:16,574 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/0ad9f7ccb0724d2280f8fe58eaa856a0 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/0ad9f7ccb0724d2280f8fe58eaa856a0 2024-11-11T20:54:16,576 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/919bc25c242546afa70c8c2c19908e33 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/919bc25c242546afa70c8c2c19908e33 2024-11-11T20:54:16,579 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/b45c2d122f4e4d1192bf262b39a7f101 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/b45c2d122f4e4d1192bf262b39a7f101 2024-11-11T20:54:16,581 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/dd5120df2dbd4c779ff5067bbc13aba2 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/A/dd5120df2dbd4c779ff5067bbc13aba2 2024-11-11T20:54:16,586 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/004bb318ec044fdcbbad4dad41601595 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/004bb318ec044fdcbbad4dad41601595 2024-11-11T20:54:16,589 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/79c2a4a362c6405888a494c08e4e01bc to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/79c2a4a362c6405888a494c08e4e01bc 2024-11-11T20:54:16,591 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/d9cea52b808c4362bd48e8e06401ee31 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/d9cea52b808c4362bd48e8e06401ee31 2024-11-11T20:54:16,593 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/e38c9cd3c548457d9c8ce0ce2ec41fc2 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/B/e38c9cd3c548457d9c8ce0ce2ec41fc2 2024-11-11T20:54:16,597 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/19089399546944e5a78302a81fceea88 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/19089399546944e5a78302a81fceea88 2024-11-11T20:54:16,599 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/5cdfc34f69804d43b4bcb6644b590cd2 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/5cdfc34f69804d43b4bcb6644b590cd2 2024-11-11T20:54:16,601 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/9483005fda5140179408f1ee10f10bed to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/9483005fda5140179408f1ee10f10bed 2024-11-11T20:54:16,603 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/c53cae5fea694d5eb61f37e9a197aada to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/C/c53cae5fea694d5eb61f37e9a197aada 2024-11-11T20:54:16,608 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/recovered.edits/538.seqid to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb/recovered.edits/538.seqid 2024-11-11T20:54:16,608 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/13af503d1b68a9ec8649b1b1c0fa50fb 2024-11-11T20:54:16,608 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T20:54:16,612 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=76, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:54:16,617 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T20:54:16,621 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T20:54:16,622 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=76, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:54:16,622 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T20:54:16,623 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731358456622"}]},"ts":"9223372036854775807"} 2024-11-11T20:54:16,625 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T20:54:16,625 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 13af503d1b68a9ec8649b1b1c0fa50fb, NAME => 'TestAcidGuarantees,,1731358424413.13af503d1b68a9ec8649b1b1c0fa50fb.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T20:54:16,625 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T20:54:16,625 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731358456625"}]},"ts":"9223372036854775807"} 2024-11-11T20:54:16,627 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T20:54:16,628 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=76, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:54:16,629 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 70 msec 2024-11-11T20:54:16,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T20:54:16,666 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T20:54:16,668 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:54:16,674 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=246 (was 248), OpenFileDescriptor=447 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=350 (was 398), ProcessCount=11 (was 11), AvailableMemoryMB=4076 (was 4458) 2024-11-11T20:54:16,682 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=246, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=350, ProcessCount=11, AvailableMemoryMB=4076 2024-11-11T20:54:16,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T20:54:16,684 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T20:54:16,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=77, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T20:54:16,686 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T20:54:16,686 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:16,686 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 77 2024-11-11T20:54:16,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-11-11T20:54:16,687 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T20:54:16,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742123_1299 (size=1061) 2024-11-11T20:54:16,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-11-11T20:54:17,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-11-11T20:54:17,098 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23 2024-11-11T20:54:17,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742124_1300 (size=53) 2024-11-11T20:54:17,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-11-11T20:54:17,509 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:54:17,510 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing dc6d436b2ab3693a9c0834e8e24f25cf, disabling compactions & flushes 2024-11-11T20:54:17,510 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:17,510 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:17,510 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. after waiting 0 ms 2024-11-11T20:54:17,510 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:17,510 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:17,510 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for dc6d436b2ab3693a9c0834e8e24f25cf: Waiting for close lock at 1731358457510Disabling compacts and flushes for region at 1731358457510Disabling writes for close at 1731358457510Writing region close event to WAL at 1731358457510Closed at 1731358457510 2024-11-11T20:54:17,513 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T20:54:17,513 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731358457513"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358457513"}]},"ts":"1731358457513"} 2024-11-11T20:54:17,516 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T20:54:17,517 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T20:54:17,518 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358457517"}]},"ts":"1731358457517"} 2024-11-11T20:54:17,519 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T20:54:17,519 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc6d436b2ab3693a9c0834e8e24f25cf, ASSIGN}] 2024-11-11T20:54:17,521 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc6d436b2ab3693a9c0834e8e24f25cf, ASSIGN 2024-11-11T20:54:17,522 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc6d436b2ab3693a9c0834e8e24f25cf, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,44669,1731358362066; forceNewPlan=false, retain=false 2024-11-11T20:54:17,673 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=dc6d436b2ab3693a9c0834e8e24f25cf, regionState=OPENING, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:17,677 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc6d436b2ab3693a9c0834e8e24f25cf, ASSIGN because future has completed 2024-11-11T20:54:17,678 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066}] 2024-11-11T20:54:17,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-11-11T20:54:17,840 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:17,840 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7752): Opening region: {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:54:17,840 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:17,840 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:54:17,841 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7794): checking encryption for dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:17,841 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(7797): checking classloading for dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:17,843 INFO [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:17,845 INFO [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T20:54:17,845 INFO [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc6d436b2ab3693a9c0834e8e24f25cf columnFamilyName A 2024-11-11T20:54:17,845 DEBUG [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:17,846 INFO [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] regionserver.HStore(327): Store=dc6d436b2ab3693a9c0834e8e24f25cf/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:54:17,846 INFO [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:17,848 INFO [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T20:54:17,848 INFO [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc6d436b2ab3693a9c0834e8e24f25cf columnFamilyName B 2024-11-11T20:54:17,848 DEBUG [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:17,849 INFO [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] regionserver.HStore(327): Store=dc6d436b2ab3693a9c0834e8e24f25cf/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:54:17,849 INFO [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:17,850 INFO [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T20:54:17,850 INFO [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc6d436b2ab3693a9c0834e8e24f25cf columnFamilyName C 2024-11-11T20:54:17,851 DEBUG [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:17,851 INFO [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] regionserver.HStore(327): Store=dc6d436b2ab3693a9c0834e8e24f25cf/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:54:17,851 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:17,851 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1038): replaying wal for dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:17,852 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:17,852 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:17,854 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1048): stopping wal replay for dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:17,854 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1060): Cleaning up temporary data for dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:17,854 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T20:54:17,855 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1093): writing seq id for dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:17,858 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:54:17,858 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1114): Opened dc6d436b2ab3693a9c0834e8e24f25cf; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60988582, jitterRate=-0.09119930863380432}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T20:54:17,858 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:17,859 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegion(1006): Region open journal for dc6d436b2ab3693a9c0834e8e24f25cf: Running coprocessor pre-open hook at 1731358457841Writing region info on filesystem at 1731358457841Initializing all the Stores at 1731358457842 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358457842Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358457842Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358457842Cleaning up temporary data from old regions at 1731358457854 (+12 ms)Running coprocessor post-open hooks at 1731358457858 (+4 ms)Region opened successfully at 1731358457859 (+1 ms) 2024-11-11T20:54:17,860 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., pid=79, masterSystemTime=1731358457833 2024-11-11T20:54:17,862 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:17,862 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=79}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:17,863 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=dc6d436b2ab3693a9c0834e8e24f25cf, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:17,865 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=78, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 because future has completed 2024-11-11T20:54:17,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-11T20:54:17,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; OpenRegionProcedure dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 in 188 msec 2024-11-11T20:54:17,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-11-11T20:54:17,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc6d436b2ab3693a9c0834e8e24f25cf, ASSIGN in 348 msec 2024-11-11T20:54:17,869 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T20:54:17,869 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358457869"}]},"ts":"1731358457869"} 2024-11-11T20:54:17,871 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T20:54:17,871 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=77, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T20:54:17,872 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1870 sec 2024-11-11T20:54:18,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-11-11T20:54:18,837 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:54:18,840 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d3de22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:18,840 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:54:18,840 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:54:18,843 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:54:18,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:54:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:54:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40c36da5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:18,844 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:54:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:54:18,845 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:18,847 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33554, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:54:18,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b3988ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:18,849 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:54:18,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:54:18,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:54:18,851 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40960, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:54:18,852 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T20:54:18,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 51ca66f7ee3c,38429,1731358361462 2024-11-11T20:54:18,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1d5112d6 2024-11-11T20:54:18,853 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T20:54:18,854 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T20:54:18,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T20:54:18,856 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T20:54:18,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T20:54:18,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742125_1301 (size=1097) 2024-11-11T20:54:19,272 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-11T20:54:19,272 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-11T20:54:19,274 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T20:54:19,277 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc6d436b2ab3693a9c0834e8e24f25cf, REOPEN/MOVE}] 2024-11-11T20:54:19,278 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc6d436b2ab3693a9c0834e8e24f25cf, REOPEN/MOVE 2024-11-11T20:54:19,279 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=dc6d436b2ab3693a9c0834e8e24f25cf, regionState=CLOSING, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:19,280 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc6d436b2ab3693a9c0834e8e24f25cf, REOPEN/MOVE because future has completed 2024-11-11T20:54:19,281 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T20:54:19,281 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066}] 2024-11-11T20:54:19,433 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(122): Close dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:19,434 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T20:54:19,434 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1722): Closing dc6d436b2ab3693a9c0834e8e24f25cf, disabling compactions & flushes 2024-11-11T20:54:19,434 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:19,434 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:19,434 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. after waiting 0 ms 2024-11-11T20:54:19,434 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:19,438 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:54:19,438 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:19,438 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegion(1676): Region close journal for dc6d436b2ab3693a9c0834e8e24f25cf: Waiting for close lock at 1731358459434Running coprocessor pre-close hooks at 1731358459434Disabling compacts and flushes for region at 1731358459434Disabling writes for close at 1731358459434Writing region close event to WAL at 1731358459434Running coprocessor post-close hooks at 1731358459438 (+4 ms)Closed at 1731358459438 2024-11-11T20:54:19,438 WARN [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] regionserver.HRegionServer(3299): Not adding moved region record: dc6d436b2ab3693a9c0834e8e24f25cf to self. 2024-11-11T20:54:19,440 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=83}] handler.UnassignRegionHandler(157): Closed dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:19,441 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=dc6d436b2ab3693a9c0834e8e24f25cf, regionState=CLOSED 2024-11-11T20:54:19,442 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=83, ppid=82, state=RUNNABLE, hasLock=false; CloseRegionProcedure dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 because future has completed 2024-11-11T20:54:19,444 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-11T20:54:19,444 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseRegionProcedure dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 in 162 msec 2024-11-11T20:54:19,445 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc6d436b2ab3693a9c0834e8e24f25cf, REOPEN/MOVE; state=CLOSED, location=51ca66f7ee3c,44669,1731358362066; forceNewPlan=false, retain=true 2024-11-11T20:54:19,596 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=dc6d436b2ab3693a9c0834e8e24f25cf, regionState=OPENING, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:19,600 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=82, ppid=81, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc6d436b2ab3693a9c0834e8e24f25cf, REOPEN/MOVE because future has completed 2024-11-11T20:54:19,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066}] 2024-11-11T20:54:19,762 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:19,762 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(7752): Opening region: {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:54:19,763 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:19,763 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:54:19,763 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(7794): checking encryption for dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:19,763 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(7797): checking classloading for dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:19,765 INFO [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:19,766 INFO [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T20:54:19,766 INFO [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc6d436b2ab3693a9c0834e8e24f25cf columnFamilyName A 2024-11-11T20:54:19,768 DEBUG [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:19,769 INFO [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] regionserver.HStore(327): Store=dc6d436b2ab3693a9c0834e8e24f25cf/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:54:19,769 INFO [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:19,770 INFO [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T20:54:19,770 INFO [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc6d436b2ab3693a9c0834e8e24f25cf columnFamilyName B 2024-11-11T20:54:19,770 DEBUG [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:19,771 INFO [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] regionserver.HStore(327): Store=dc6d436b2ab3693a9c0834e8e24f25cf/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:54:19,771 INFO [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:19,772 INFO [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T20:54:19,772 INFO [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dc6d436b2ab3693a9c0834e8e24f25cf columnFamilyName C 2024-11-11T20:54:19,772 DEBUG [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:19,773 INFO [StoreOpener-dc6d436b2ab3693a9c0834e8e24f25cf-1 {}] regionserver.HStore(327): Store=dc6d436b2ab3693a9c0834e8e24f25cf/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:54:19,773 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:19,773 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1038): replaying wal for dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:19,773 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:19,774 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:19,775 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1048): stopping wal replay for dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:19,775 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1060): Cleaning up temporary data for dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:19,776 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T20:54:19,777 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1093): writing seq id for dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:19,777 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1114): Opened dc6d436b2ab3693a9c0834e8e24f25cf; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59587080, jitterRate=-0.1120833158493042}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T20:54:19,777 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1122): Running coprocessor post-open hooks for dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:19,778 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegion(1006): Region open journal for dc6d436b2ab3693a9c0834e8e24f25cf: Running coprocessor pre-open hook at 1731358459763Writing region info on filesystem at 1731358459763Initializing all the Stores at 1731358459764 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358459765 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358459765Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358459765Cleaning up temporary data from old regions at 1731358459775 (+10 ms)Running coprocessor post-open hooks at 1731358459777 (+2 ms)Region opened successfully at 1731358459778 (+1 ms) 2024-11-11T20:54:19,778 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., pid=84, masterSystemTime=1731358459754 2024-11-11T20:54:19,780 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:19,780 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=84}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:19,780 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=82 updating hbase:meta row=dc6d436b2ab3693a9c0834e8e24f25cf, regionState=OPEN, openSeqNum=5, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:19,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=82, state=RUNNABLE, hasLock=false; OpenRegionProcedure dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 because future has completed 2024-11-11T20:54:19,783 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=82 2024-11-11T20:54:19,783 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=82, state=SUCCESS, hasLock=false; OpenRegionProcedure dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 in 181 msec 2024-11-11T20:54:19,785 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-11T20:54:19,785 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc6d436b2ab3693a9c0834e8e24f25cf, REOPEN/MOVE in 506 msec 2024-11-11T20:54:19,787 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-11T20:54:19,787 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 511 msec 2024-11-11T20:54:19,789 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 931 msec 2024-11-11T20:54:19,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T20:54:19,790 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:54:19,790 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c3d4f84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,790 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:54:19,790 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:54:19,791 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:54:19,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:54:19,791 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:54:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d8735c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:54:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:54:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:19,793 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44924, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:54:19,793 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b99a2a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,794 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ad3f2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,794 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:54:19,794 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:54:19,795 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:54:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:54:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:54:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29ffd69a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:54:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:54:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:19,796 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:54:19,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fa1910d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,797 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@422fcba0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,797 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:54:19,797 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:54:19,798 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:54:19,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:54:19,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:54:19,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54ae01e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:54:19,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:54:19,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:19,799 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44972, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:54:19,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52737a0e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,800 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72e71a40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,800 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:54:19,800 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:54:19,801 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:54:19,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:54:19,801 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:54:19,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5288757b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:54:19,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:54:19,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:19,803 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:54:19,803 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2408cd19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,804 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b9dbe2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,804 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:54:19,804 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:54:19,804 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:54:19,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:54:19,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:54:19,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fbcae43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:54:19,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:54:19,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:19,806 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:54:19,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5faadb4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,807 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10e2b5a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,807 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:54:19,807 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:54:19,808 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:54:19,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:54:19,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:54:19,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1151b5ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:54:19,808 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:54:19,809 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:19,809 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:54:19,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@645f274d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,810 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d41a701, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,810 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:54:19,810 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:54:19,810 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:54:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:54:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:54:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d074659, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:54:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:54:19,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:19,811 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:54:19,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c8d883f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,812 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2f46ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,812 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:54:19,812 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:54:19,813 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:54:19,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:54:19,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:54:19,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7090e93e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:54:19,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:54:19,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:19,814 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:54:19,814 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dffd523, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,815 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180bd544, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,815 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:54:19,815 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:54:19,816 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:54:19,816 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:54:19,816 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:54:19,816 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17f63200, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,816 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:54:19,816 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:54:19,816 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:19,817 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:54:19,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@abc6b29, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,818 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d553196, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,818 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:54:19,818 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:54:19,819 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:54:19,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:54:19,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:54:19,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73e7cf2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:54:19,819 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:54:19,820 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:19,820 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45090, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:54:19,821 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67cfa12f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:19,822 DEBUG [Thread-1409 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:54:19,822 DEBUG [Thread-1394 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:54:19,822 DEBUG [Thread-1411 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:54:19,822 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:54:19,822 DEBUG [Thread-1407 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:54:19,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:54:19,823 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:54:19,823 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:54:19,823 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:54:19,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:54:19,823 DEBUG [Thread-1396 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:54:19,823 DEBUG [Thread-1402 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:54:19,823 DEBUG [Thread-1400 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:54:19,823 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:54:19,824 DEBUG [Thread-1398 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:54:19,824 DEBUG [Thread-1413 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:54:19,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:54:19,824 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:54:19,824 DEBUG [Thread-1405 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:54:19,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees 2024-11-11T20:54:19,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:54:19,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:54:19,825 DEBUG [Thread-1402 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:54:19,825 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47550, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:54:19,825 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47536, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:54:19,825 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47556, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:54:19,825 DEBUG [Thread-1402 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:54:19,825 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:54:19,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:54:19,825 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47570, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:54:19,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-11T20:54:19,826 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:54:19,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:54:19,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:54:19,826 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:54:19,826 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:54:19,826 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:54:19,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:54:19,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:54:19,826 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:54:19,827 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:54:19,827 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:54:19,828 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47600, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:54:19,828 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47616, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:54:19,828 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:54:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:54:19,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:54:19,830 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47630, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:54:19,830 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:54:19,831 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:54:19,832 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:54:19,833 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:54:19,833 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:54:19,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:54:19,835 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:54:19,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:54:19,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:19,842 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dc6d436b2ab3693a9c0834e8e24f25cf 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T20:54:19,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=A 2024-11-11T20:54:19,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:19,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=B 2024-11-11T20:54:19,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:19,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=C 2024-11-11T20:54:19,843 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:19,870 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110a6a9b5d9f114d219cc57352a6c8a1b9_dc6d436b2ab3693a9c0834e8e24f25cf is 50, key is test_row_0/A:col10/1731358459839/Put/seqid=0 2024-11-11T20:54:19,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742126_1302 (size=14594) 2024-11-11T20:54:19,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:19,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47600 deadline: 1731358519880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:19,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:19,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47606 deadline: 1731358519882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:19,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:19,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:19,889 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:19,889 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:19,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:19,890 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:19,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358459878, max timestamp=1731358459887Num uniques 859; 2024-11-11T20:54:19,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358459878, max timestamp=1731358459887Num uniques 859; 2024-11-11T20:54:19,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:54:19,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:54:19,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358459878, max timestamp=1731358459887Num uniques 859; 2024-11-11T20:54:19,890 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:54:19,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-11T20:54:19,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:19,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358519950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:19,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:19,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:19,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:19,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358519950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:19,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:19,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:19,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:19,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:19,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358519951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:19,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:19,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:19,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:19,963 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:19,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-11T20:54:19,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:19,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:19,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:19,982 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:19,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:19,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:20,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:20,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47600 deadline: 1731358519996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:20,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47606 deadline: 1731358519996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:20,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:20,071 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:20,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358520066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:20,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:20,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358520066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:20,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:20,072 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:20,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358520066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:20,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:20,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,072 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:20,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-11T20:54:20,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:20,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:20,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:20,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:20,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:20,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:20,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-11T20:54:20,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:20,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47600 deadline: 1731358520206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:20,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:20,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47606 deadline: 1731358520206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:20,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:20,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:20,279 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:20,280 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:20,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358520276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:20,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:20,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358520276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:20,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:20,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:20,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,280 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:20,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358520276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:20,283 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110a6a9b5d9f114d219cc57352a6c8a1b9_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110a6a9b5d9f114d219cc57352a6c8a1b9_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:20,283 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/188138385396484780189e66800f670f, store: [table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:20,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/188138385396484780189e66800f670f is 175, key is test_row_0/A:col10/1731358459839/Put/seqid=0 2024-11-11T20:54:20,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742127_1303 (size=39545) 2024-11-11T20:54:20,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-11T20:54:20,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:20,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:20,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:20,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:20,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:20,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:20,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-11T20:54:20,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:20,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:20,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:20,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:20,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:20,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:20,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-11T20:54:20,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:20,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47600 deadline: 1731358520526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:20,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:20,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47606 deadline: 1731358520526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:20,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,528 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:20,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,528 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:20,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:20,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358520586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:20,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358520586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:20,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358520586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:20,591 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:20,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-11T20:54:20,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:20,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:20,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:20,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:20,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:20,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:20,687 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/188138385396484780189e66800f670f 2024-11-11T20:54:20,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/bca58eb419764460afef7595bde276a5 is 50, key is test_row_0/B:col10/1731358459839/Put/seqid=0 2024-11-11T20:54:20,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742128_1304 (size=9657) 2024-11-11T20:54:20,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-11T20:54:20,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:20,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:20,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:20,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:20,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:20,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:20,897 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-11T20:54:20,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:20,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:20,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:20,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:20,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:20,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:20,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-11T20:54:21,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47606 deadline: 1731358521035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:21,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:21,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:21,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:21,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47600 deadline: 1731358521035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:21,042 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:21,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:21,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:21,043 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:21,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-11T20:54:21,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:21,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:21,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:21,050 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:21,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:21,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:21,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:21,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358521095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:21,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:21,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358521095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:21,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:21,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358521095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:21,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/bca58eb419764460afef7595bde276a5 2024-11-11T20:54:21,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/c7492faa43944475a8650e334f2b6f6b is 50, key is test_row_0/C:col10/1731358459839/Put/seqid=0 2024-11-11T20:54:21,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742129_1305 (size=9657) 2024-11-11T20:54:21,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-11T20:54:21,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:21,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:21,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:21,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:21,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:21,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:21,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-11T20:54:21,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:21,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:21,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:21,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:21,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:21,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:21,443 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T20:54:21,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-11T20:54:21,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:21,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:21,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:21,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:21,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:21,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:21,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/c7492faa43944475a8650e334f2b6f6b 2024-11-11T20:54:21,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/188138385396484780189e66800f670f as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/188138385396484780189e66800f670f 2024-11-11T20:54:21,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/188138385396484780189e66800f670f, entries=200, sequenceid=15, filesize=38.6 K 2024-11-11T20:54:21,547 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/bca58eb419764460afef7595bde276a5 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/bca58eb419764460afef7595bde276a5 2024-11-11T20:54:21,551 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/bca58eb419764460afef7595bde276a5, entries=100, sequenceid=15, filesize=9.4 K 2024-11-11T20:54:21,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/c7492faa43944475a8650e334f2b6f6b as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/c7492faa43944475a8650e334f2b6f6b 2024-11-11T20:54:21,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/c7492faa43944475a8650e334f2b6f6b, entries=100, sequenceid=15, filesize=9.4 K 2024-11-11T20:54:21,556 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for dc6d436b2ab3693a9c0834e8e24f25cf in 1714ms, sequenceid=15, compaction requested=false 2024-11-11T20:54:21,556 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:21,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-11T20:54:21,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:21,661 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2902): Flushing dc6d436b2ab3693a9c0834e8e24f25cf 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-11T20:54:21,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=A 2024-11-11T20:54:21,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:54:21,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=B 2024-11-11T20:54:21,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:54:21,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=C 2024-11-11T20:54:21,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:54:21,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d341b64bb0944de5af1deeefe3cf823c_dc6d436b2ab3693a9c0834e8e24f25cf is 50, key is test_row_0/A:col10/1731358459912/Put/seqid=0 2024-11-11T20:54:21,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742130_1306 (size=12154) 2024-11-11T20:54:21,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-11T20:54:22,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:22,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:22,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:22,076 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d341b64bb0944de5af1deeefe3cf823c_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d341b64bb0944de5af1deeefe3cf823c_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:22,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/8a53a8b38cd34715a95966edbe2aee7a, store: [table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:22,077 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/8a53a8b38cd34715a95966edbe2aee7a is 175, key is test_row_0/A:col10/1731358459912/Put/seqid=0 2024-11-11T20:54:22,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742131_1307 (size=30955) 2024-11-11T20:54:22,081 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/8a53a8b38cd34715a95966edbe2aee7a 2024-11-11T20:54:22,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/7b723a1064394268b4bf6b890d4514f3 is 50, key is test_row_0/B:col10/1731358459912/Put/seqid=0 2024-11-11T20:54:22,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:22,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47606 deadline: 1731358522084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:22,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:22,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742132_1308 (size=12001) 2024-11-11T20:54:22,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:22,092 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:22,093 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/7b723a1064394268b4bf6b890d4514f3 2024-11-11T20:54:22,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:22,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47600 deadline: 1731358522089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:22,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:22,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:22,095 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:22,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/15aeb2171f63446990ba28a70e582a26 is 50, key is test_row_0/C:col10/1731358459912/Put/seqid=0 2024-11-11T20:54:22,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742133_1309 (size=12001) 2024-11-11T20:54:22,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358522115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:22,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358522115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:22,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358522115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:22,123 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_1' in dc6d436b2ab3693a9c0834e8e24f25cf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:54:22,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:22,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:22,124 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:22,123 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_1' in dc6d436b2ab3693a9c0834e8e24f25cf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:54:22,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:22,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:22,124 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:22,124 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in dc6d436b2ab3693a9c0834e8e24f25cf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:54:22,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:22,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:22,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:22,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:22,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47606 deadline: 1731358522195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:22,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:22,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:22,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:22,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:22,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47600 deadline: 1731358522205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:22,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:22,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:22,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:22,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47606 deadline: 1731358522406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:22,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:22,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:22,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:22,420 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:22,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47600 deadline: 1731358522415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:22,421 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:22,507 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/15aeb2171f63446990ba28a70e582a26 2024-11-11T20:54:22,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/8a53a8b38cd34715a95966edbe2aee7a as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/8a53a8b38cd34715a95966edbe2aee7a 2024-11-11T20:54:22,515 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/8a53a8b38cd34715a95966edbe2aee7a, entries=150, sequenceid=51, filesize=30.2 K 2024-11-11T20:54:22,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/7b723a1064394268b4bf6b890d4514f3 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/7b723a1064394268b4bf6b890d4514f3 2024-11-11T20:54:22,519 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/7b723a1064394268b4bf6b890d4514f3, entries=150, sequenceid=51, filesize=11.7 K 2024-11-11T20:54:22,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/15aeb2171f63446990ba28a70e582a26 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/15aeb2171f63446990ba28a70e582a26 2024-11-11T20:54:22,525 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/15aeb2171f63446990ba28a70e582a26, entries=150, sequenceid=51, filesize=11.7 K 2024-11-11T20:54:22,526 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for dc6d436b2ab3693a9c0834e8e24f25cf in 865ms, sequenceid=51, compaction requested=false 2024-11-11T20:54:22,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2603): Flush status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:22,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:22,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=86 2024-11-11T20:54:22,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=86 2024-11-11T20:54:22,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-11T20:54:22,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7000 sec 2024-11-11T20:54:22,530 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees in 2.7070 sec 2024-11-11T20:54:22,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:22,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dc6d436b2ab3693a9c0834e8e24f25cf 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T20:54:22,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=A 2024-11-11T20:54:22,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:22,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=B 2024-11-11T20:54:22,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:22,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=C 2024-11-11T20:54:22,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:22,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117f1dbf16a05f41cd9385172bdf1c1783_dc6d436b2ab3693a9c0834e8e24f25cf is 50, key is test_row_0/A:col10/1731358462088/Put/seqid=0 2024-11-11T20:54:22,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742134_1310 (size=14594) 2024-11-11T20:54:22,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:22,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47600 deadline: 1731358522786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:22,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:22,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:22,795 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:22,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:22,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47606 deadline: 1731358522792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:22,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:22,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:22,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:22,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47600 deadline: 1731358522905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:22,907 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:22,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:22,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:22,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47606 deadline: 1731358522905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:22,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:22,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:22,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:22,908 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:23,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47600 deadline: 1731358523116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:23,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47606 deadline: 1731358523116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:23,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:23,131 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:23,134 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117f1dbf16a05f41cd9385172bdf1c1783_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117f1dbf16a05f41cd9385172bdf1c1783_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:23,135 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/266d5b9cc4a04d10838fea0dee88b65d, store: [table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:23,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/266d5b9cc4a04d10838fea0dee88b65d is 175, key is test_row_0/A:col10/1731358462088/Put/seqid=0 2024-11-11T20:54:23,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742135_1311 (size=39549) 2024-11-11T20:54:23,139 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/266d5b9cc4a04d10838fea0dee88b65d 2024-11-11T20:54:23,151 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/df8e43f1d1944d36805c23a3d4c32e68 is 50, key is test_row_0/B:col10/1731358462088/Put/seqid=0 2024-11-11T20:54:23,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742136_1312 (size=12001) 2024-11-11T20:54:23,443 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:23,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47600 deadline: 1731358523436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:23,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:23,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:23,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47606 deadline: 1731358523436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:23,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:23,444 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:23,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:23,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:23,444 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:23,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/df8e43f1d1944d36805c23a3d4c32e68 2024-11-11T20:54:23,560 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/cd1df2c0a49a49d0960ab715ef285cc3 is 50, key is test_row_0/C:col10/1731358462088/Put/seqid=0 2024-11-11T20:54:23,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742137_1313 (size=12001) 2024-11-11T20:54:23,964 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/cd1df2c0a49a49d0960ab715ef285cc3 2024-11-11T20:54:23,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:23,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47600 deadline: 1731358523956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:23,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:23,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:23,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47606 deadline: 1731358523956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:23,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:23,965 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:23,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:23,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:23,965 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:23,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/266d5b9cc4a04d10838fea0dee88b65d as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/266d5b9cc4a04d10838fea0dee88b65d 2024-11-11T20:54:23,971 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/266d5b9cc4a04d10838fea0dee88b65d, entries=200, sequenceid=63, filesize=38.6 K 2024-11-11T20:54:23,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/df8e43f1d1944d36805c23a3d4c32e68 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/df8e43f1d1944d36805c23a3d4c32e68 2024-11-11T20:54:23,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/df8e43f1d1944d36805c23a3d4c32e68, entries=150, sequenceid=63, filesize=11.7 K 2024-11-11T20:54:23,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/cd1df2c0a49a49d0960ab715ef285cc3 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/cd1df2c0a49a49d0960ab715ef285cc3 2024-11-11T20:54:23,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/cd1df2c0a49a49d0960ab715ef285cc3, entries=150, sequenceid=63, filesize=11.7 K 2024-11-11T20:54:23,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for dc6d436b2ab3693a9c0834e8e24f25cf in 1263ms, sequenceid=63, compaction requested=true 2024-11-11T20:54:23,984 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:23,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc6d436b2ab3693a9c0834e8e24f25cf:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:54:23,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:23,984 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:54:23,984 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:54:23,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc6d436b2ab3693a9c0834e8e24f25cf:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:54:23,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:23,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc6d436b2ab3693a9c0834e8e24f25cf:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:54:23,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:54:23,986 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:54:23,986 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): dc6d436b2ab3693a9c0834e8e24f25cf/B is initiating minor compaction (all files) 2024-11-11T20:54:23,986 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc6d436b2ab3693a9c0834e8e24f25cf/B in TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:23,987 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/bca58eb419764460afef7595bde276a5, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/7b723a1064394268b4bf6b890d4514f3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/df8e43f1d1944d36805c23a3d4c32e68] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp, totalSize=32.9 K 2024-11-11T20:54:23,988 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:54:23,988 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): dc6d436b2ab3693a9c0834e8e24f25cf/A is initiating minor compaction (all files) 2024-11-11T20:54:23,988 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc6d436b2ab3693a9c0834e8e24f25cf/A in TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:23,988 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/188138385396484780189e66800f670f, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/8a53a8b38cd34715a95966edbe2aee7a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/266d5b9cc4a04d10838fea0dee88b65d] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp, totalSize=107.5 K 2024-11-11T20:54:23,988 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:23,988 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. files: [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/188138385396484780189e66800f670f, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/8a53a8b38cd34715a95966edbe2aee7a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/266d5b9cc4a04d10838fea0dee88b65d] 2024-11-11T20:54:23,988 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting bca58eb419764460afef7595bde276a5, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731358459839 2024-11-11T20:54:23,989 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 188138385396484780189e66800f670f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731358459834 2024-11-11T20:54:23,989 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 7b723a1064394268b4bf6b890d4514f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731358459912 2024-11-11T20:54:23,989 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a53a8b38cd34715a95966edbe2aee7a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731358459912 2024-11-11T20:54:23,989 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting df8e43f1d1944d36805c23a3d4c32e68, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731358462082 2024-11-11T20:54:23,990 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 266d5b9cc4a04d10838fea0dee88b65d, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731358462067 2024-11-11T20:54:23,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-11T20:54:23,996 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:54:23,996 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:54:23,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-11-11T20:54:23,998 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:54:23,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T20:54:23,999 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:54:23,999 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:54:24,010 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:24,012 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc6d436b2ab3693a9c0834e8e24f25cf#B#compaction#265 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:24,013 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/81e835a8a6a5491f853a7136a746da64 is 50, key is test_row_0/B:col10/1731358462088/Put/seqid=0 2024-11-11T20:54:24,017 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111ed46fa87d13649c8839f5bb1a416efcd_dc6d436b2ab3693a9c0834e8e24f25cf store=[table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:24,019 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111ed46fa87d13649c8839f5bb1a416efcd_dc6d436b2ab3693a9c0834e8e24f25cf, store=[table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:24,021 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ed46fa87d13649c8839f5bb1a416efcd_dc6d436b2ab3693a9c0834e8e24f25cf because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:24,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742139_1315 (size=4469) 2024-11-11T20:54:24,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742138_1314 (size=12104) 2024-11-11T20:54:24,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T20:54:24,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:24,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dc6d436b2ab3693a9c0834e8e24f25cf 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-11T20:54:24,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=A 2024-11-11T20:54:24,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-11T20:54:24,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:54:24,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=B 2024-11-11T20:54:24,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:54:24,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=C 2024-11-11T20:54:24,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:54:24,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:24,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:24,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:24,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:24,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:24,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:24,160 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c11d62cc346e4d748a85d16179a660fd_dc6d436b2ab3693a9c0834e8e24f25cf is 50, key is test_row_0/A:col10/1731358462783/Put/seqid=0 2024-11-11T20:54:24,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742140_1316 (size=12154) 2024-11-11T20:54:24,165 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:24,171 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c11d62cc346e4d748a85d16179a660fd_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c11d62cc346e4d748a85d16179a660fd_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:24,172 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/40d1b6b19b7046a5bda3571439b2b4b7, store: [table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:24,173 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/40d1b6b19b7046a5bda3571439b2b4b7 is 175, key is test_row_0/A:col10/1731358462783/Put/seqid=0 2024-11-11T20:54:24,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742141_1317 (size=30955) 2024-11-11T20:54:24,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:24,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358524168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:24,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:24,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:24,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:24,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:24,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358524169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:24,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:24,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:24,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:24,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358524179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:24,185 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:24,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:24,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:24,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358524286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:24,295 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:24,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:24,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358524295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:24,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:24,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:24,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358524295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:24,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:24,303 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:24,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:24,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:24,303 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:24,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-11T20:54:24,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:24,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:24,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:24,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:24,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:24,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:24,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T20:54:24,432 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc6d436b2ab3693a9c0834e8e24f25cf#A#compaction#264 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:24,433 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/98c364f002f9417e87c828ecefa17ada is 175, key is test_row_0/A:col10/1731358462088/Put/seqid=0 2024-11-11T20:54:24,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742142_1318 (size=31058) 2024-11-11T20:54:24,441 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/81e835a8a6a5491f853a7136a746da64 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/81e835a8a6a5491f853a7136a746da64 2024-11-11T20:54:24,446 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dc6d436b2ab3693a9c0834e8e24f25cf/B of dc6d436b2ab3693a9c0834e8e24f25cf into 81e835a8a6a5491f853a7136a746da64(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:24,446 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:24,446 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., storeName=dc6d436b2ab3693a9c0834e8e24f25cf/B, priority=13, startTime=1731358463984; duration=0sec 2024-11-11T20:54:24,446 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:54:24,446 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc6d436b2ab3693a9c0834e8e24f25cf:B 2024-11-11T20:54:24,446 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:54:24,447 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:54:24,447 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): dc6d436b2ab3693a9c0834e8e24f25cf/C is initiating minor compaction (all files) 2024-11-11T20:54:24,447 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc6d436b2ab3693a9c0834e8e24f25cf/C in TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:24,448 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/c7492faa43944475a8650e334f2b6f6b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/15aeb2171f63446990ba28a70e582a26, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/cd1df2c0a49a49d0960ab715ef285cc3] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp, totalSize=32.9 K 2024-11-11T20:54:24,448 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting c7492faa43944475a8650e334f2b6f6b, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731358459839 2024-11-11T20:54:24,448 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 15aeb2171f63446990ba28a70e582a26, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731358459912 2024-11-11T20:54:24,448 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting cd1df2c0a49a49d0960ab715ef285cc3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731358462082 2024-11-11T20:54:24,459 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc6d436b2ab3693a9c0834e8e24f25cf#C#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:24,459 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/82ee1638a9044e02b90cf79a0d8140ae is 50, key is test_row_0/C:col10/1731358462088/Put/seqid=0 2024-11-11T20:54:24,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-11T20:54:24,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:24,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:24,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:24,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:24,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:24,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:24,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742143_1319 (size=12104) 2024-11-11T20:54:24,467 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/82ee1638a9044e02b90cf79a0d8140ae as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/82ee1638a9044e02b90cf79a0d8140ae 2024-11-11T20:54:24,474 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dc6d436b2ab3693a9c0834e8e24f25cf/C of dc6d436b2ab3693a9c0834e8e24f25cf into 82ee1638a9044e02b90cf79a0d8140ae(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:24,474 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:24,474 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., storeName=dc6d436b2ab3693a9c0834e8e24f25cf/C, priority=13, startTime=1731358463985; duration=0sec 2024-11-11T20:54:24,474 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:24,474 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc6d436b2ab3693a9c0834e8e24f25cf:C 2024-11-11T20:54:24,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358524506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:24,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:24,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358524506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:24,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:24,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:24,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:24,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:24,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:24,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358524506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:24,512 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:24,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:24,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:24,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:24,577 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/40d1b6b19b7046a5bda3571439b2b4b7 2024-11-11T20:54:24,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/61172cbbf9e24987af07bed4ad8fbaaa is 50, key is test_row_0/B:col10/1731358462783/Put/seqid=0 2024-11-11T20:54:24,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742144_1320 (size=12001) 2024-11-11T20:54:24,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-11T20:54:24,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:24,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:24,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:24,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:24,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T20:54:24,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-11T20:54:24,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:24,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:24,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:24,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:24,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:24,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358524816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:24,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:24,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358524816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:24,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:24,822 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:24,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:24,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:24,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:24,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:24,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358524826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:24,833 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:24,840 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/98c364f002f9417e87c828ecefa17ada as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/98c364f002f9417e87c828ecefa17ada 2024-11-11T20:54:24,844 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dc6d436b2ab3693a9c0834e8e24f25cf/A of dc6d436b2ab3693a9c0834e8e24f25cf into 98c364f002f9417e87c828ecefa17ada(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:24,844 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:24,844 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., storeName=dc6d436b2ab3693a9c0834e8e24f25cf/A, priority=13, startTime=1731358463984; duration=0sec 2024-11-11T20:54:24,844 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:24,844 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc6d436b2ab3693a9c0834e8e24f25cf:A 2024-11-11T20:54:24,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-11T20:54:24,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:24,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:24,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:24,918 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:24,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:24,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:24,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47600 deadline: 1731358524975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:24,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:24,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47606 deadline: 1731358524975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:24,980 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_1' in dc6d436b2ab3693a9c0834e8e24f25cf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:54:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:24,981 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in dc6d436b2ab3693a9c0834e8e24f25cf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:54:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:24,981 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:24,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/61172cbbf9e24987af07bed4ad8fbaaa 2024-11-11T20:54:25,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/3fac554d9a034750a1c16cee0d812434 is 50, key is test_row_0/C:col10/1731358462783/Put/seqid=0 2024-11-11T20:54:25,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742145_1321 (size=12001) 2024-11-11T20:54:25,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-11T20:54:25,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:25,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:25,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:25,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:25,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:25,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:25,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T20:54:25,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-11T20:54:25,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:25,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:25,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:25,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:25,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:25,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:25,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:25,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358525335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:25,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:25,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:25,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:25,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358525335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:25,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:25,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:25,342 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:25,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:25,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358525335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:25,342 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:25,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:25,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:25,343 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:25,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-11T20:54:25,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:25,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:25,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:25,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] handler.RSProcedureHandler(58): pid=88 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:25,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=88 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:25,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=88 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:25,408 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/3fac554d9a034750a1c16cee0d812434 2024-11-11T20:54:25,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/40d1b6b19b7046a5bda3571439b2b4b7 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/40d1b6b19b7046a5bda3571439b2b4b7 2024-11-11T20:54:25,415 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/40d1b6b19b7046a5bda3571439b2b4b7, entries=150, sequenceid=84, filesize=30.2 K 2024-11-11T20:54:25,416 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/61172cbbf9e24987af07bed4ad8fbaaa as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/61172cbbf9e24987af07bed4ad8fbaaa 2024-11-11T20:54:25,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,419 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/61172cbbf9e24987af07bed4ad8fbaaa, entries=150, sequenceid=84, filesize=11.7 K 2024-11-11T20:54:25,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/3fac554d9a034750a1c16cee0d812434 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/3fac554d9a034750a1c16cee0d812434 2024-11-11T20:54:25,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/3fac554d9a034750a1c16cee0d812434, entries=150, sequenceid=84, filesize=11.7 K 2024-11-11T20:54:25,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,424 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for dc6d436b2ab3693a9c0834e8e24f25cf in 1272ms, sequenceid=84, compaction requested=false 2024-11-11T20:54:25,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:25,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-11T20:54:25,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:25,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,528 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2902): Flushing dc6d436b2ab3693a9c0834e8e24f25cf 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T20:54:25,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=A 2024-11-11T20:54:25,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:25,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=B 2024-11-11T20:54:25,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:25,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=C 2024-11-11T20:54:25,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:25,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a23d76584770430cb73ff30a021d81ad_dc6d436b2ab3693a9c0834e8e24f25cf is 50, key is test_row_1/A:col10/1731358464168/Put/seqid=0 2024-11-11T20:54:25,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742146_1322 (size=9714) 2024-11-11T20:54:25,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,959 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a23d76584770430cb73ff30a021d81ad_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a23d76584770430cb73ff30a021d81ad_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:25,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/0b8fd0cb1acb42379547ebbd39ef97f1, store: [table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:25,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/0b8fd0cb1acb42379547ebbd39ef97f1 is 175, key is test_row_1/A:col10/1731358464168/Put/seqid=0 2024-11-11T20:54:25,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742147_1323 (size=22361) 2024-11-11T20:54:25,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T20:54:26,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,367 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/0b8fd0cb1acb42379547ebbd39ef97f1 2024-11-11T20:54:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/64aacecff2c144f3bc5e5353e8ffc89f is 50, key is test_row_1/B:col10/1731358464168/Put/seqid=0 2024-11-11T20:54:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:26,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742148_1324 (size=9657) 2024-11-11T20:54:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:26,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358466456, max timestamp=1731358466543Num uniques 859; 2024-11-11T20:54:26,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358466456, max timestamp=1731358466543Num uniques 859; 2024-11-11T20:54:26,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:54:26,546 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:54:26,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358466456, max timestamp=1731358466543Num uniques 859; 2024-11-11T20:54:26,547 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:54:26,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358526620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:26,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:26,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:26,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:26,629 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358526621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:26,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:26,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:26,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:26,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:26,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358526622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:26,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:26,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:26,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:26,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:26,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358526735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:26,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358526735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:26,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:26,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358526735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:26,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:26,782 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/64aacecff2c144f3bc5e5353e8ffc89f 2024-11-11T20:54:26,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/ada68df1a6d346f09e627c4bbb51e253 is 50, key is test_row_1/C:col10/1731358464168/Put/seqid=0 2024-11-11T20:54:26,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742149_1325 (size=9657) 2024-11-11T20:54:26,963 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:26,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358526955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:26,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:26,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:26,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:26,964 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358526955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:26,964 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:26,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:26,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:26,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358526955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:26,964 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:26,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:26,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:26,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:26,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:26,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47600 deadline: 1731358526986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:26,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47606 deadline: 1731358526986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:26,991 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_1' in dc6d436b2ab3693a9c0834e8e24f25cf of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:54:26,991 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in dc6d436b2ab3693a9c0834e8e24f25cf of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:54:26,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:26,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:26,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:26,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:26,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:26,991 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:27,192 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/ada68df1a6d346f09e627c4bbb51e253 2024-11-11T20:54:27,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/0b8fd0cb1acb42379547ebbd39ef97f1 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/0b8fd0cb1acb42379547ebbd39ef97f1 2024-11-11T20:54:27,199 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/0b8fd0cb1acb42379547ebbd39ef97f1, entries=100, sequenceid=98, filesize=21.8 K 2024-11-11T20:54:27,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/64aacecff2c144f3bc5e5353e8ffc89f as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/64aacecff2c144f3bc5e5353e8ffc89f 2024-11-11T20:54:27,202 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/64aacecff2c144f3bc5e5353e8ffc89f, entries=100, sequenceid=98, filesize=9.4 K 2024-11-11T20:54:27,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/ada68df1a6d346f09e627c4bbb51e253 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/ada68df1a6d346f09e627c4bbb51e253 2024-11-11T20:54:27,206 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/ada68df1a6d346f09e627c4bbb51e253, entries=100, sequenceid=98, filesize=9.4 K 2024-11-11T20:54:27,207 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for dc6d436b2ab3693a9c0834e8e24f25cf in 1679ms, sequenceid=98, compaction requested=true 2024-11-11T20:54:27,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2603): Flush status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:27,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:27,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-11-11T20:54:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-11-11T20:54:27,210 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-11T20:54:27,210 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2090 sec 2024-11-11T20:54:27,211 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 3.2140 sec 2024-11-11T20:54:27,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:27,272 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dc6d436b2ab3693a9c0834e8e24f25cf 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-11T20:54:27,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=A 2024-11-11T20:54:27,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:54:27,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=B 2024-11-11T20:54:27,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:54:27,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=C 2024-11-11T20:54:27,273 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:54:27,279 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e7369472e7bf488e8a356db26099c149_dc6d436b2ab3693a9c0834e8e24f25cf is 50, key is test_row_0/A:col10/1731358466621/Put/seqid=0 2024-11-11T20:54:27,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742150_1326 (size=14744) 2024-11-11T20:54:27,282 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:27,285 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e7369472e7bf488e8a356db26099c149_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e7369472e7bf488e8a356db26099c149_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:27,286 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/a166b1910a634cd28606825025e1b64b, store: [table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:27,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/a166b1910a634cd28606825025e1b64b is 175, key is test_row_0/A:col10/1731358466621/Put/seqid=0 2024-11-11T20:54:27,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742151_1327 (size=39699) 2024-11-11T20:54:27,294 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/a166b1910a634cd28606825025e1b64b 2024-11-11T20:54:27,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/a1c83f8b5a0946f1b0f4d15cad66e0a8 is 50, key is test_row_0/B:col10/1731358466621/Put/seqid=0 2024-11-11T20:54:27,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742152_1328 (size=12151) 2024-11-11T20:54:27,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358527307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:27,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:27,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:27,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:27,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:27,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358527307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:27,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:27,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:27,310 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:27,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358527308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:27,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:27,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:27,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:27,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:27,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358527415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:27,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:27,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:27,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:27,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:27,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358527416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:27,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:27,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:27,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:27,432 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:27,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358527426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:27,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:27,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:27,433 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:27,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358527635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:27,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:27,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:27,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358527636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:27,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:27,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:27,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:27,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358527636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:27,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:27,641 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:27,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:27,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:27,641 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:27,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/a1c83f8b5a0946f1b0f4d15cad66e0a8 2024-11-11T20:54:27,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/d3e1933f3e2e4dc08721f11dc089b253 is 50, key is test_row_0/C:col10/1731358466621/Put/seqid=0 2024-11-11T20:54:27,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742153_1329 (size=12151) 2024-11-11T20:54:27,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358527946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:27,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:27,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358527946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:27,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:27,957 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:27,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:27,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358527946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:27,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:27,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:27,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:28,115 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/d3e1933f3e2e4dc08721f11dc089b253 2024-11-11T20:54:28,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/a166b1910a634cd28606825025e1b64b as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/a166b1910a634cd28606825025e1b64b 2024-11-11T20:54:28,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/a166b1910a634cd28606825025e1b64b, entries=200, sequenceid=134, filesize=38.8 K 2024-11-11T20:54:28,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/a1c83f8b5a0946f1b0f4d15cad66e0a8 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/a1c83f8b5a0946f1b0f4d15cad66e0a8 2024-11-11T20:54:28,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/a1c83f8b5a0946f1b0f4d15cad66e0a8, entries=150, sequenceid=134, filesize=11.9 K 2024-11-11T20:54:28,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/d3e1933f3e2e4dc08721f11dc089b253 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/d3e1933f3e2e4dc08721f11dc089b253 2024-11-11T20:54:28,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/d3e1933f3e2e4dc08721f11dc089b253, entries=150, sequenceid=134, filesize=11.9 K 2024-11-11T20:54:28,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for dc6d436b2ab3693a9c0834e8e24f25cf in 859ms, sequenceid=134, compaction requested=true 2024-11-11T20:54:28,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:28,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc6d436b2ab3693a9c0834e8e24f25cf:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:54:28,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:28,131 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:54:28,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc6d436b2ab3693a9c0834e8e24f25cf:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:54:28,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:28,131 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:54:28,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc6d436b2ab3693a9c0834e8e24f25cf:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:54:28,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:54:28,132 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124073 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:54:28,132 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45913 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:54:28,132 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): dc6d436b2ab3693a9c0834e8e24f25cf/A is initiating minor compaction (all files) 2024-11-11T20:54:28,132 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): dc6d436b2ab3693a9c0834e8e24f25cf/B is initiating minor compaction (all files) 2024-11-11T20:54:28,132 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc6d436b2ab3693a9c0834e8e24f25cf/B in TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:28,132 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc6d436b2ab3693a9c0834e8e24f25cf/A in TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:28,132 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/98c364f002f9417e87c828ecefa17ada, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/40d1b6b19b7046a5bda3571439b2b4b7, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/0b8fd0cb1acb42379547ebbd39ef97f1, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/a166b1910a634cd28606825025e1b64b] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp, totalSize=121.2 K 2024-11-11T20:54:28,132 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/81e835a8a6a5491f853a7136a746da64, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/61172cbbf9e24987af07bed4ad8fbaaa, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/64aacecff2c144f3bc5e5353e8ffc89f, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/a1c83f8b5a0946f1b0f4d15cad66e0a8] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp, totalSize=44.8 K 2024-11-11T20:54:28,132 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:28,132 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. files: [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/98c364f002f9417e87c828ecefa17ada, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/40d1b6b19b7046a5bda3571439b2b4b7, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/0b8fd0cb1acb42379547ebbd39ef97f1, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/a166b1910a634cd28606825025e1b64b] 2024-11-11T20:54:28,133 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 81e835a8a6a5491f853a7136a746da64, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731358462082 2024-11-11T20:54:28,133 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 98c364f002f9417e87c828ecefa17ada, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731358462082 2024-11-11T20:54:28,133 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 61172cbbf9e24987af07bed4ad8fbaaa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731358462783 2024-11-11T20:54:28,133 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40d1b6b19b7046a5bda3571439b2b4b7, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731358462783 2024-11-11T20:54:28,133 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 64aacecff2c144f3bc5e5353e8ffc89f, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731358464168 2024-11-11T20:54:28,133 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b8fd0cb1acb42379547ebbd39ef97f1, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731358464168 2024-11-11T20:54:28,134 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting a1c83f8b5a0946f1b0f4d15cad66e0a8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731358466620 2024-11-11T20:54:28,134 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting a166b1910a634cd28606825025e1b64b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731358466605 2024-11-11T20:54:28,144 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc6d436b2ab3693a9c0834e8e24f25cf#B#compaction#276 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:28,145 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/7c55d131c3d6467299ec6fe65f4364f1 is 50, key is test_row_0/B:col10/1731358466621/Put/seqid=0 2024-11-11T20:54:28,150 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:28,152 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411113958a2d522ca45038c7f5c880bf78e62_dc6d436b2ab3693a9c0834e8e24f25cf store=[table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:28,154 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411113958a2d522ca45038c7f5c880bf78e62_dc6d436b2ab3693a9c0834e8e24f25cf, store=[table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:28,154 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113958a2d522ca45038c7f5c880bf78e62_dc6d436b2ab3693a9c0834e8e24f25cf because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:28,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742154_1330 (size=12391) 2024-11-11T20:54:28,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742155_1331 (size=4469) 2024-11-11T20:54:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T20:54:28,165 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:54:28,166 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:54:28,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-11T20:54:28,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-11T20:54:28,167 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:54:28,168 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:54:28,168 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:54:28,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-11T20:54:28,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-11T20:54:28,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:28,320 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing dc6d436b2ab3693a9c0834e8e24f25cf 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T20:54:28,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=A 2024-11-11T20:54:28,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:28,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=B 2024-11-11T20:54:28,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:28,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=C 2024-11-11T20:54:28,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:28,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112117c6f0853d4b39a7411853f04e5123_dc6d436b2ab3693a9c0834e8e24f25cf is 50, key is test_row_0/A:col10/1731358467306/Put/seqid=0 2024-11-11T20:54:28,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742156_1332 (size=12304) 2024-11-11T20:54:28,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:28,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:28,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-11T20:54:28,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731358468500, max timestamp=1731358468537Num uniques 859; 2024-11-11T20:54:28,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T20:54:28,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731358468500, max timestamp=1731358468537Num uniques 859; 2024-11-11T20:54:28,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T20:54:28,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731358468500, max timestamp=1731358468537Num uniques 859; 2024-11-11T20:54:28,540 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T20:54:28,559 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/7c55d131c3d6467299ec6fe65f4364f1 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/7c55d131c3d6467299ec6fe65f4364f1 2024-11-11T20:54:28,563 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dc6d436b2ab3693a9c0834e8e24f25cf/B of dc6d436b2ab3693a9c0834e8e24f25cf into 7c55d131c3d6467299ec6fe65f4364f1(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:28,563 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:28,563 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., storeName=dc6d436b2ab3693a9c0834e8e24f25cf/B, priority=12, startTime=1731358468131; duration=0sec 2024-11-11T20:54:28,563 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:54:28,563 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc6d436b2ab3693a9c0834e8e24f25cf:B 2024-11-11T20:54:28,563 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:54:28,564 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc6d436b2ab3693a9c0834e8e24f25cf#A#compaction#277 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:28,564 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/5681fa02631044759a68c1c40b8c009f is 175, key is test_row_0/A:col10/1731358466621/Put/seqid=0 2024-11-11T20:54:28,565 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45913 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:54:28,565 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): dc6d436b2ab3693a9c0834e8e24f25cf/C is initiating minor compaction (all files) 2024-11-11T20:54:28,565 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc6d436b2ab3693a9c0834e8e24f25cf/C in TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:28,566 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/82ee1638a9044e02b90cf79a0d8140ae, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/3fac554d9a034750a1c16cee0d812434, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/ada68df1a6d346f09e627c4bbb51e253, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/d3e1933f3e2e4dc08721f11dc089b253] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp, totalSize=44.8 K 2024-11-11T20:54:28,566 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 82ee1638a9044e02b90cf79a0d8140ae, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731358462082 2024-11-11T20:54:28,566 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fac554d9a034750a1c16cee0d812434, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731358462783 2024-11-11T20:54:28,567 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting ada68df1a6d346f09e627c4bbb51e253, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731358464168 2024-11-11T20:54:28,567 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting d3e1933f3e2e4dc08721f11dc089b253, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731358466620 2024-11-11T20:54:28,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742157_1333 (size=31345) 2024-11-11T20:54:28,576 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/5681fa02631044759a68c1c40b8c009f as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/5681fa02631044759a68c1c40b8c009f 2024-11-11T20:54:28,584 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dc6d436b2ab3693a9c0834e8e24f25cf/A of dc6d436b2ab3693a9c0834e8e24f25cf into 5681fa02631044759a68c1c40b8c009f(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:28,584 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:28,584 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., storeName=dc6d436b2ab3693a9c0834e8e24f25cf/A, priority=12, startTime=1731358468131; duration=0sec 2024-11-11T20:54:28,584 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:28,584 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc6d436b2ab3693a9c0834e8e24f25cf:A 2024-11-11T20:54:28,588 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc6d436b2ab3693a9c0834e8e24f25cf#C#compaction#279 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:28,588 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/ff60676ded3a4de184888b5aea5df04a is 50, key is test_row_0/C:col10/1731358466621/Put/seqid=0 2024-11-11T20:54:28,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742158_1334 (size=12391) 2024-11-11T20:54:28,604 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:28,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358528593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:28,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:28,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:28,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:28,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:28,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358528596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:28,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:28,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:28,610 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:28,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:28,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358528597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:28,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:28,614 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:28,615 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:28,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358528716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:28,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:28,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358528716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:28,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:28,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:28,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:28,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:28,726 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:28,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:28,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358528725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:28,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:28,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:28,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:28,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:28,737 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112117c6f0853d4b39a7411853f04e5123_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112117c6f0853d4b39a7411853f04e5123_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:28,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/544b0ecb79d94b1da8b5fc1c5554dae9, store: [table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:28,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/544b0ecb79d94b1da8b5fc1c5554dae9 is 175, key is test_row_0/A:col10/1731358467306/Put/seqid=0 2024-11-11T20:54:28,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742159_1335 (size=31105) 2024-11-11T20:54:28,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-11T20:54:28,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:28,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358528936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:28,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:28,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358528936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:28,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:28,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358528936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:28,943 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:28,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:28,998 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/ff60676ded3a4de184888b5aea5df04a as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/ff60676ded3a4de184888b5aea5df04a 2024-11-11T20:54:29,001 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dc6d436b2ab3693a9c0834e8e24f25cf/C of dc6d436b2ab3693a9c0834e8e24f25cf into ff60676ded3a4de184888b5aea5df04a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:29,001 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:29,001 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., storeName=dc6d436b2ab3693a9c0834e8e24f25cf/C, priority=12, startTime=1731358468131; duration=0sec 2024-11-11T20:54:29,002 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:29,002 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc6d436b2ab3693a9c0834e8e24f25cf:C 2024-11-11T20:54:29,143 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=144, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/544b0ecb79d94b1da8b5fc1c5554dae9 2024-11-11T20:54:29,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/ffbad171e8c04b17923d97d119a4f20a is 50, key is test_row_0/B:col10/1731358467306/Put/seqid=0 2024-11-11T20:54:29,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742160_1336 (size=12151) 2024-11-11T20:54:29,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:29,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358529245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:29,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:29,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:29,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:29,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358529256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:29,264 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:29,264 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:29,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358529256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:29,265 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:29,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-11T20:54:29,553 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/ffbad171e8c04b17923d97d119a4f20a 2024-11-11T20:54:29,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/4e396caddf5741798a6d240c8f331d0d is 50, key is test_row_0/C:col10/1731358467306/Put/seqid=0 2024-11-11T20:54:29,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742161_1337 (size=12151) 2024-11-11T20:54:29,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:29,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358529765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:29,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:29,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:29,770 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:29,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:29,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358529776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:29,781 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:29,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:29,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358529776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:29,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:29,782 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:29,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:29,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:29,782 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:29,975 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/4e396caddf5741798a6d240c8f331d0d 2024-11-11T20:54:29,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/544b0ecb79d94b1da8b5fc1c5554dae9 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/544b0ecb79d94b1da8b5fc1c5554dae9 2024-11-11T20:54:29,981 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/544b0ecb79d94b1da8b5fc1c5554dae9, entries=150, sequenceid=144, filesize=30.4 K 2024-11-11T20:54:29,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/ffbad171e8c04b17923d97d119a4f20a as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/ffbad171e8c04b17923d97d119a4f20a 2024-11-11T20:54:29,985 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/ffbad171e8c04b17923d97d119a4f20a, entries=150, sequenceid=144, filesize=11.9 K 2024-11-11T20:54:29,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/4e396caddf5741798a6d240c8f331d0d as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/4e396caddf5741798a6d240c8f331d0d 2024-11-11T20:54:29,989 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/4e396caddf5741798a6d240c8f331d0d, entries=150, sequenceid=144, filesize=11.9 K 2024-11-11T20:54:29,989 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for dc6d436b2ab3693a9c0834e8e24f25cf in 1669ms, sequenceid=144, compaction requested=false 2024-11-11T20:54:29,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:29,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:29,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-11T20:54:29,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-11T20:54:29,992 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-11T20:54:29,992 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8230 sec 2024-11-11T20:54:29,994 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 1.8270 sec 2024-11-11T20:54:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-11T20:54:30,316 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:54:30,316 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:54:30,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-11T20:54:30,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T20:54:30,318 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:54:30,319 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:54:30,319 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:54:30,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T20:54:30,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-11T20:54:30,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:30,471 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing dc6d436b2ab3693a9c0834e8e24f25cf 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-11T20:54:30,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=A 2024-11-11T20:54:30,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:54:30,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=B 2024-11-11T20:54:30,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:54:30,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=C 2024-11-11T20:54:30,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:54:30,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119cb2c34196184b4ebe40cb9db5ec214c_dc6d436b2ab3693a9c0834e8e24f25cf is 50, key is test_row_0/A:col10/1731358468594/Put/seqid=0 2024-11-11T20:54:30,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742162_1338 (size=12304) 2024-11-11T20:54:30,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:30,497 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119cb2c34196184b4ebe40cb9db5ec214c_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119cb2c34196184b4ebe40cb9db5ec214c_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:30,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/ce5afa35f2c94867965666813810a1ec, store: [table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:30,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/ce5afa35f2c94867965666813810a1ec is 175, key is test_row_0/A:col10/1731358468594/Put/seqid=0 2024-11-11T20:54:30,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742163_1339 (size=31105) 2024-11-11T20:54:30,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T20:54:30,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:30,791 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:30,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:30,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358530805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:30,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:30,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:30,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:30,816 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:30,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358530806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:30,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:30,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:30,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:30,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:30,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358530814, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:30,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:30,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:30,821 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:30,903 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=182, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/ce5afa35f2c94867965666813810a1ec 2024-11-11T20:54:30,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/110a1c95728441c2a3ca70da4f7f03be is 50, key is test_row_0/B:col10/1731358468594/Put/seqid=0 2024-11-11T20:54:30,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742164_1340 (size=12151) 2024-11-11T20:54:30,912 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/110a1c95728441c2a3ca70da4f7f03be 2024-11-11T20:54:30,918 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/95e7120748354e95b89e2acb07456790 is 50, key is test_row_0/C:col10/1731358468594/Put/seqid=0 2024-11-11T20:54:30,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742165_1341 (size=12151) 2024-11-11T20:54:30,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:30,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358530926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:30,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:30,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358530926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:30,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:30,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358530926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:30,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:30,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:30,934 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:30,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T20:54:31,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47600 deadline: 1731358531016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:31,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:31,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47606 deadline: 1731358531016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:31,022 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_1' in dc6d436b2ab3693a9c0834e8e24f25cf of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:54:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:31,022 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in dc6d436b2ab3693a9c0834e8e24f25cf of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:54:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:31,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:31,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358531135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:31,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:31,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:31,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:31,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358531135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:31,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:31,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:31,141 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:31,141 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358531135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:31,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:31,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:31,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:31,322 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=182 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/95e7120748354e95b89e2acb07456790 2024-11-11T20:54:31,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/ce5afa35f2c94867965666813810a1ec as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/ce5afa35f2c94867965666813810a1ec 2024-11-11T20:54:31,329 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/ce5afa35f2c94867965666813810a1ec, entries=150, sequenceid=182, filesize=30.4 K 2024-11-11T20:54:31,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/110a1c95728441c2a3ca70da4f7f03be as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/110a1c95728441c2a3ca70da4f7f03be 2024-11-11T20:54:31,333 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/110a1c95728441c2a3ca70da4f7f03be, entries=150, sequenceid=182, filesize=11.9 K 2024-11-11T20:54:31,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/95e7120748354e95b89e2acb07456790 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/95e7120748354e95b89e2acb07456790 2024-11-11T20:54:31,337 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/95e7120748354e95b89e2acb07456790, entries=150, sequenceid=182, filesize=11.9 K 2024-11-11T20:54:31,338 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for dc6d436b2ab3693a9c0834e8e24f25cf in 867ms, sequenceid=182, compaction requested=true 2024-11-11T20:54:31,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:31,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:31,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-11T20:54:31,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-11T20:54:31,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-11T20:54:31,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0210 sec 2024-11-11T20:54:31,344 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 1.0270 sec 2024-11-11T20:54:31,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:31,449 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dc6d436b2ab3693a9c0834e8e24f25cf 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T20:54:31,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=A 2024-11-11T20:54:31,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:31,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=B 2024-11-11T20:54:31,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:31,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=C 2024-11-11T20:54:31,450 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:31,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-11T20:54:31,455 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:54:31,456 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:54:31,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a2bbfd3a950142b0a635883e5f0f0128_dc6d436b2ab3693a9c0834e8e24f25cf is 50, key is test_row_0/A:col10/1731358471447/Put/seqid=0 2024-11-11T20:54:31,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-11T20:54:31,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T20:54:31,463 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:54:31,464 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:54:31,464 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:54:31,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742166_1342 (size=17284) 2024-11-11T20:54:31,514 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:31,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358531503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:31,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:31,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:31,514 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:31,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:31,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358531513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:31,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:31,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:31,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:31,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:31,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358531514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:31,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:31,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:31,525 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T20:54:31,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T20:54:31,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:31,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:31,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:31,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:31,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:31,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:31,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358531626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:31,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:31,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:31,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358531626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:31,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:31,632 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:31,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:31,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:31,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:31,644 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:31,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358531636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:31,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:31,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:31,644 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:31,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T20:54:31,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:31,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:31,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:31,769 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:31,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:31,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:31,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T20:54:31,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:31,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358531836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:31,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:31,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:31,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:31,845 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:31,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358531836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:31,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:31,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:31,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:31,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:31,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358531855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:31,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:31,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:31,863 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:31,864 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:31,868 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a2bbfd3a950142b0a635883e5f0f0128_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a2bbfd3a950142b0a635883e5f0f0128_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:31,868 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/22bbf26497e1477ca7cd440778307fe2, store: [table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:31,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/22bbf26497e1477ca7cd440778307fe2 is 175, key is test_row_0/A:col10/1731358471447/Put/seqid=0 2024-11-11T20:54:31,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742167_1343 (size=48389) 2024-11-11T20:54:31,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T20:54:31,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:31,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:31,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:31,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:31,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:31,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:32,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T20:54:32,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:32,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:32,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:32,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:32,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:32,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:32,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T20:54:32,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:32,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358532155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:32,164 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:32,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358532155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:32,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:32,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:32,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358532176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:32,183 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:32,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:32,184 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:32,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T20:54:32,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:32,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:32,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:32,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:32,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:32,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:32,273 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/22bbf26497e1477ca7cd440778307fe2 2024-11-11T20:54:32,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/269d3578f0744da4a2a1f2d0036b2c3b is 50, key is test_row_0/B:col10/1731358471447/Put/seqid=0 2024-11-11T20:54:32,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742168_1344 (size=12151) 2024-11-11T20:54:32,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T20:54:32,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:32,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:32,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:32,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:32,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:32,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:32,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T20:54:32,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:32,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:32,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:32,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:32,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:32,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:32,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T20:54:32,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/269d3578f0744da4a2a1f2d0036b2c3b 2024-11-11T20:54:32,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T20:54:32,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:32,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:32,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:32,685 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:32,685 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:32,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:32,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358532676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:32,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:32,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358532676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:32,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:32,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:32,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:32,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:32,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:32,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:32,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:32,689 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/b97dc3d2f701441c9c0c96852625ee0d is 50, key is test_row_0/C:col10/1731358471447/Put/seqid=0 2024-11-11T20:54:32,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742169_1345 (size=12151) 2024-11-11T20:54:32,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:32,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358532695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:32,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:32,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:32,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:32,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T20:54:32,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:32,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:32,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:32,838 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:32,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:32,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:32,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T20:54:32,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:32,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:32,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:32,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:32,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:32,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:33,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/b97dc3d2f701441c9c0c96852625ee0d 2024-11-11T20:54:33,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/22bbf26497e1477ca7cd440778307fe2 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/22bbf26497e1477ca7cd440778307fe2 2024-11-11T20:54:33,104 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/22bbf26497e1477ca7cd440778307fe2, entries=250, sequenceid=194, filesize=47.3 K 2024-11-11T20:54:33,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/269d3578f0744da4a2a1f2d0036b2c3b as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/269d3578f0744da4a2a1f2d0036b2c3b 2024-11-11T20:54:33,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/269d3578f0744da4a2a1f2d0036b2c3b, entries=150, sequenceid=194, filesize=11.9 K 2024-11-11T20:54:33,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/b97dc3d2f701441c9c0c96852625ee0d as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/b97dc3d2f701441c9c0c96852625ee0d 2024-11-11T20:54:33,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/b97dc3d2f701441c9c0c96852625ee0d, entries=150, sequenceid=194, filesize=11.9 K 2024-11-11T20:54:33,112 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for dc6d436b2ab3693a9c0834e8e24f25cf in 1663ms, sequenceid=194, compaction requested=true 2024-11-11T20:54:33,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:33,112 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:54:33,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc6d436b2ab3693a9c0834e8e24f25cf:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:54:33,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:33,112 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:54:33,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc6d436b2ab3693a9c0834e8e24f25cf:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:54:33,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:33,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc6d436b2ab3693a9c0834e8e24f25cf:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:54:33,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:54:33,113 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141944 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:54:33,113 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): dc6d436b2ab3693a9c0834e8e24f25cf/A is initiating minor compaction (all files) 2024-11-11T20:54:33,113 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:54:33,113 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc6d436b2ab3693a9c0834e8e24f25cf/A in TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:33,113 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): dc6d436b2ab3693a9c0834e8e24f25cf/B is initiating minor compaction (all files) 2024-11-11T20:54:33,113 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/5681fa02631044759a68c1c40b8c009f, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/544b0ecb79d94b1da8b5fc1c5554dae9, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/ce5afa35f2c94867965666813810a1ec, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/22bbf26497e1477ca7cd440778307fe2] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp, totalSize=138.6 K 2024-11-11T20:54:33,113 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc6d436b2ab3693a9c0834e8e24f25cf/B in TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:33,113 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:33,113 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. files: [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/5681fa02631044759a68c1c40b8c009f, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/544b0ecb79d94b1da8b5fc1c5554dae9, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/ce5afa35f2c94867965666813810a1ec, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/22bbf26497e1477ca7cd440778307fe2] 2024-11-11T20:54:33,113 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/7c55d131c3d6467299ec6fe65f4364f1, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/ffbad171e8c04b17923d97d119a4f20a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/110a1c95728441c2a3ca70da4f7f03be, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/269d3578f0744da4a2a1f2d0036b2c3b] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp, totalSize=47.7 K 2024-11-11T20:54:33,114 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5681fa02631044759a68c1c40b8c009f, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731358466620 2024-11-11T20:54:33,114 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c55d131c3d6467299ec6fe65f4364f1, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731358466620 2024-11-11T20:54:33,114 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 544b0ecb79d94b1da8b5fc1c5554dae9, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731358467306 2024-11-11T20:54:33,114 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting ffbad171e8c04b17923d97d119a4f20a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731358467306 2024-11-11T20:54:33,114 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce5afa35f2c94867965666813810a1ec, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731358468591 2024-11-11T20:54:33,114 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 110a1c95728441c2a3ca70da4f7f03be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731358468591 2024-11-11T20:54:33,114 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22bbf26497e1477ca7cd440778307fe2, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731358470792 2024-11-11T20:54:33,114 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 269d3578f0744da4a2a1f2d0036b2c3b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731358470813 2024-11-11T20:54:33,124 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc6d436b2ab3693a9c0834e8e24f25cf#B#compaction#288 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:33,124 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/e0666c896d9a47d381e3cee283d1500a is 50, key is test_row_0/B:col10/1731358471447/Put/seqid=0 2024-11-11T20:54:33,128 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:33,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742170_1346 (size=12527) 2024-11-11T20:54:33,137 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111420e0620680d45e99f7ac484f1fd081e_dc6d436b2ab3693a9c0834e8e24f25cf store=[table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:33,140 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111420e0620680d45e99f7ac484f1fd081e_dc6d436b2ab3693a9c0834e8e24f25cf, store=[table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:33,141 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111420e0620680d45e99f7ac484f1fd081e_dc6d436b2ab3693a9c0834e8e24f25cf because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:33,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-11T20:54:33,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:33,143 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing dc6d436b2ab3693a9c0834e8e24f25cf 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T20:54:33,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=A 2024-11-11T20:54:33,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:33,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=B 2024-11-11T20:54:33,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:33,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=C 2024-11-11T20:54:33,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:33,148 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742171_1347 (size=4469) 2024-11-11T20:54:33,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f92c9c6a6dd046f19d2fd9166bfacdb5_dc6d436b2ab3693a9c0834e8e24f25cf is 50, key is test_row_0/A:col10/1731358471500/Put/seqid=0 2024-11-11T20:54:33,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742172_1348 (size=12304) 2024-11-11T20:54:33,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:33,163 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f92c9c6a6dd046f19d2fd9166bfacdb5_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f92c9c6a6dd046f19d2fd9166bfacdb5_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:33,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/6b23ecd846ee442c914e1b65ff0f5d73, store: [table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:33,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/6b23ecd846ee442c914e1b65ff0f5d73 is 175, key is test_row_0/A:col10/1731358471500/Put/seqid=0 2024-11-11T20:54:33,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742173_1349 (size=31105) 2024-11-11T20:54:33,539 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/e0666c896d9a47d381e3cee283d1500a as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/e0666c896d9a47d381e3cee283d1500a 2024-11-11T20:54:33,543 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dc6d436b2ab3693a9c0834e8e24f25cf/B of dc6d436b2ab3693a9c0834e8e24f25cf into e0666c896d9a47d381e3cee283d1500a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:33,543 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:33,543 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., storeName=dc6d436b2ab3693a9c0834e8e24f25cf/B, priority=12, startTime=1731358473112; duration=0sec 2024-11-11T20:54:33,543 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:54:33,543 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc6d436b2ab3693a9c0834e8e24f25cf:B 2024-11-11T20:54:33,543 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:54:33,544 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48844 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:54:33,544 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): dc6d436b2ab3693a9c0834e8e24f25cf/C is initiating minor compaction (all files) 2024-11-11T20:54:33,544 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc6d436b2ab3693a9c0834e8e24f25cf/C in TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:33,544 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/ff60676ded3a4de184888b5aea5df04a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/4e396caddf5741798a6d240c8f331d0d, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/95e7120748354e95b89e2acb07456790, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/b97dc3d2f701441c9c0c96852625ee0d] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp, totalSize=47.7 K 2024-11-11T20:54:33,544 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting ff60676ded3a4de184888b5aea5df04a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731358466620 2024-11-11T20:54:33,545 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e396caddf5741798a6d240c8f331d0d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731358467306 2024-11-11T20:54:33,545 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 95e7120748354e95b89e2acb07456790, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=182, earliestPutTs=1731358468591 2024-11-11T20:54:33,545 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting b97dc3d2f701441c9c0c96852625ee0d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731358470813 2024-11-11T20:54:33,548 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc6d436b2ab3693a9c0834e8e24f25cf#A#compaction#289 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:33,549 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/e383557d462147b88207bd12c13df32a is 175, key is test_row_0/A:col10/1731358471447/Put/seqid=0 2024-11-11T20:54:33,552 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc6d436b2ab3693a9c0834e8e24f25cf#C#compaction#291 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:33,553 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/ad8c182967a345e8ae18d89fcb97ffb1 is 50, key is test_row_0/C:col10/1731358471447/Put/seqid=0 2024-11-11T20:54:33,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742174_1350 (size=31481) 2024-11-11T20:54:33,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742175_1351 (size=12527) 2024-11-11T20:54:33,568 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/6b23ecd846ee442c914e1b65ff0f5d73 2024-11-11T20:54:33,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/9d8fc233e80b4a548903a1d97b06321e is 50, key is test_row_0/B:col10/1731358471500/Put/seqid=0 2024-11-11T20:54:33,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742176_1352 (size=12151) 2024-11-11T20:54:33,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T20:54:33,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:33,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:33,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:33,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358533725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:33,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:33,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:33,733 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:33,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:33,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358533730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:33,738 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:33,743 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:33,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358533731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:33,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:33,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:33,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:33,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358533835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:33,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:33,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:33,839 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:33,853 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:33,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358533845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:33,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:33,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:33,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:33,854 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:33,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358533845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:33,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:33,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:33,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:33,958 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/e383557d462147b88207bd12c13df32a as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/e383557d462147b88207bd12c13df32a 2024-11-11T20:54:33,959 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/ad8c182967a345e8ae18d89fcb97ffb1 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/ad8c182967a345e8ae18d89fcb97ffb1 2024-11-11T20:54:33,962 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dc6d436b2ab3693a9c0834e8e24f25cf/A of dc6d436b2ab3693a9c0834e8e24f25cf into e383557d462147b88207bd12c13df32a(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:33,962 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:33,962 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., storeName=dc6d436b2ab3693a9c0834e8e24f25cf/A, priority=12, startTime=1731358473112; duration=0sec 2024-11-11T20:54:33,962 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:33,962 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc6d436b2ab3693a9c0834e8e24f25cf:A 2024-11-11T20:54:33,965 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dc6d436b2ab3693a9c0834e8e24f25cf/C of dc6d436b2ab3693a9c0834e8e24f25cf into ad8c182967a345e8ae18d89fcb97ffb1(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:33,966 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:33,966 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., storeName=dc6d436b2ab3693a9c0834e8e24f25cf/C, priority=12, startTime=1731358473113; duration=0sec 2024-11-11T20:54:33,966 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:33,966 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc6d436b2ab3693a9c0834e8e24f25cf:C 2024-11-11T20:54:33,979 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/9d8fc233e80b4a548903a1d97b06321e 2024-11-11T20:54:33,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/ecdf3824e68d4523ac223c58eb1b517e is 50, key is test_row_0/C:col10/1731358471500/Put/seqid=0 2024-11-11T20:54:33,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742177_1353 (size=12151) 2024-11-11T20:54:34,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:34,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358534045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:34,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:34,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:34,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358534055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:34,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:34,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:34,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:34,070 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:34,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358534065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:34,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:34,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:34,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:34,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:34,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358534356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:34,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:34,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:34,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:34,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:34,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358534366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:34,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:34,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:34,372 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:34,381 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:34,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358534375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:34,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:34,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:34,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:34,389 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/ecdf3824e68d4523ac223c58eb1b517e 2024-11-11T20:54:34,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/6b23ecd846ee442c914e1b65ff0f5d73 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/6b23ecd846ee442c914e1b65ff0f5d73 2024-11-11T20:54:34,398 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/6b23ecd846ee442c914e1b65ff0f5d73, entries=150, sequenceid=214, filesize=30.4 K 2024-11-11T20:54:34,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/9d8fc233e80b4a548903a1d97b06321e as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/9d8fc233e80b4a548903a1d97b06321e 2024-11-11T20:54:34,401 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/9d8fc233e80b4a548903a1d97b06321e, entries=150, sequenceid=214, filesize=11.9 K 2024-11-11T20:54:34,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/ecdf3824e68d4523ac223c58eb1b517e as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/ecdf3824e68d4523ac223c58eb1b517e 2024-11-11T20:54:34,405 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/ecdf3824e68d4523ac223c58eb1b517e, entries=150, sequenceid=214, filesize=11.9 K 2024-11-11T20:54:34,406 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for dc6d436b2ab3693a9c0834e8e24f25cf in 1262ms, sequenceid=214, compaction requested=false 2024-11-11T20:54:34,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:34,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:34,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-11T20:54:34,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-11T20:54:34,408 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-11T20:54:34,408 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9430 sec 2024-11-11T20:54:34,410 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.9530 sec 2024-11-11T20:54:34,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:34,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dc6d436b2ab3693a9c0834e8e24f25cf 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T20:54:34,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=A 2024-11-11T20:54:34,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:34,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=B 2024-11-11T20:54:34,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:34,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=C 2024-11-11T20:54:34,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:34,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115c18b26b65be47ecbaafea28f5921fd4_dc6d436b2ab3693a9c0834e8e24f25cf is 50, key is test_row_0/A:col10/1731358473711/Put/seqid=0 2024-11-11T20:54:34,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742178_1354 (size=14794) 2024-11-11T20:54:34,880 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:34,883 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115c18b26b65be47ecbaafea28f5921fd4_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115c18b26b65be47ecbaafea28f5921fd4_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:34,883 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/6f6096e1a86b4bd4b1499bd8c972833c, store: [table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:34,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/6f6096e1a86b4bd4b1499bd8c972833c is 175, key is test_row_0/A:col10/1731358473711/Put/seqid=0 2024-11-11T20:54:34,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742179_1355 (size=39749) 2024-11-11T20:54:34,933 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:34,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358534932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:34,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:34,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:34,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:34,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358534933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:34,944 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:34,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:34,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358534933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:34,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:34,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:34,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:34,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:34,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:35,040 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358535035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:35,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,040 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:35,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:35,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358535046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:35,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,051 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:35,061 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:35,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358535056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:35,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,061 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:35,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:35,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358535245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:35,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,249 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:35,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:35,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358535256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:35,276 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:35,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358535266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,277 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:35,292 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=230, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/6f6096e1a86b4bd4b1499bd8c972833c 2024-11-11T20:54:35,297 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/f575aee3bf024717a6f5c0de7903bb80 is 50, key is test_row_0/B:col10/1731358473711/Put/seqid=0 2024-11-11T20:54:35,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742180_1356 (size=12151) 2024-11-11T20:54:35,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/f575aee3bf024717a6f5c0de7903bb80 2024-11-11T20:54:35,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/6d0e2fd88a8e4ef0a8e7af6b203ff51a is 50, key is test_row_0/C:col10/1731358473711/Put/seqid=0 2024-11-11T20:54:35,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742181_1357 (size=12151) 2024-11-11T20:54:35,329 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/6d0e2fd88a8e4ef0a8e7af6b203ff51a 2024-11-11T20:54:35,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/6f6096e1a86b4bd4b1499bd8c972833c as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/6f6096e1a86b4bd4b1499bd8c972833c 2024-11-11T20:54:35,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/6f6096e1a86b4bd4b1499bd8c972833c, entries=200, sequenceid=230, filesize=38.8 K 2024-11-11T20:54:35,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/f575aee3bf024717a6f5c0de7903bb80 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/f575aee3bf024717a6f5c0de7903bb80 2024-11-11T20:54:35,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/f575aee3bf024717a6f5c0de7903bb80, entries=150, sequenceid=230, filesize=11.9 K 2024-11-11T20:54:35,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/6d0e2fd88a8e4ef0a8e7af6b203ff51a as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/6d0e2fd88a8e4ef0a8e7af6b203ff51a 2024-11-11T20:54:35,355 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/6d0e2fd88a8e4ef0a8e7af6b203ff51a, entries=150, sequenceid=230, filesize=11.9 K 2024-11-11T20:54:35,356 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for dc6d436b2ab3693a9c0834e8e24f25cf in 487ms, sequenceid=230, compaction requested=true 2024-11-11T20:54:35,356 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:35,356 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:54:35,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc6d436b2ab3693a9c0834e8e24f25cf:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:54:35,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:35,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc6d436b2ab3693a9c0834e8e24f25cf:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:54:35,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:35,356 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:54:35,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc6d436b2ab3693a9c0834e8e24f25cf:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:54:35,356 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:54:35,357 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102335 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:54:35,357 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): dc6d436b2ab3693a9c0834e8e24f25cf/A is initiating minor compaction (all files) 2024-11-11T20:54:35,357 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc6d436b2ab3693a9c0834e8e24f25cf/A in TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:35,357 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:54:35,357 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/e383557d462147b88207bd12c13df32a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/6b23ecd846ee442c914e1b65ff0f5d73, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/6f6096e1a86b4bd4b1499bd8c972833c] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp, totalSize=99.9 K 2024-11-11T20:54:35,357 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): dc6d436b2ab3693a9c0834e8e24f25cf/B is initiating minor compaction (all files) 2024-11-11T20:54:35,357 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:35,357 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc6d436b2ab3693a9c0834e8e24f25cf/B in TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:35,357 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. files: [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/e383557d462147b88207bd12c13df32a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/6b23ecd846ee442c914e1b65ff0f5d73, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/6f6096e1a86b4bd4b1499bd8c972833c] 2024-11-11T20:54:35,358 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/e0666c896d9a47d381e3cee283d1500a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/9d8fc233e80b4a548903a1d97b06321e, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/f575aee3bf024717a6f5c0de7903bb80] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp, totalSize=36.0 K 2024-11-11T20:54:35,360 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting e383557d462147b88207bd12c13df32a, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731358470813 2024-11-11T20:54:35,360 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting e0666c896d9a47d381e3cee283d1500a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731358470813 2024-11-11T20:54:35,360 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d8fc233e80b4a548903a1d97b06321e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731358471500 2024-11-11T20:54:35,360 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b23ecd846ee442c914e1b65ff0f5d73, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731358471500 2024-11-11T20:54:35,360 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting f575aee3bf024717a6f5c0de7903bb80, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731358473711 2024-11-11T20:54:35,361 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f6096e1a86b4bd4b1499bd8c972833c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731358473711 2024-11-11T20:54:35,366 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:35,367 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc6d436b2ab3693a9c0834e8e24f25cf#B#compaction#297 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:35,368 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/97f998660fad417eb29b86c37ad271ba is 50, key is test_row_0/B:col10/1731358473711/Put/seqid=0 2024-11-11T20:54:35,372 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111cc5f4c5cec444947a0c32390dfbb760b_dc6d436b2ab3693a9c0834e8e24f25cf store=[table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:35,373 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111cc5f4c5cec444947a0c32390dfbb760b_dc6d436b2ab3693a9c0834e8e24f25cf, store=[table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:35,373 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111cc5f4c5cec444947a0c32390dfbb760b_dc6d436b2ab3693a9c0834e8e24f25cf because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:35,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742182_1358 (size=12629) 2024-11-11T20:54:35,385 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/97f998660fad417eb29b86c37ad271ba as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/97f998660fad417eb29b86c37ad271ba 2024-11-11T20:54:35,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742183_1359 (size=4469) 2024-11-11T20:54:35,391 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dc6d436b2ab3693a9c0834e8e24f25cf/B of dc6d436b2ab3693a9c0834e8e24f25cf into 97f998660fad417eb29b86c37ad271ba(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:35,391 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:35,391 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., storeName=dc6d436b2ab3693a9c0834e8e24f25cf/B, priority=13, startTime=1731358475356; duration=0sec 2024-11-11T20:54:35,391 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:54:35,391 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc6d436b2ab3693a9c0834e8e24f25cf:B 2024-11-11T20:54:35,391 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:54:35,392 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:54:35,392 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): dc6d436b2ab3693a9c0834e8e24f25cf/C is initiating minor compaction (all files) 2024-11-11T20:54:35,392 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc6d436b2ab3693a9c0834e8e24f25cf/C in TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:35,392 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/ad8c182967a345e8ae18d89fcb97ffb1, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/ecdf3824e68d4523ac223c58eb1b517e, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/6d0e2fd88a8e4ef0a8e7af6b203ff51a] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp, totalSize=36.0 K 2024-11-11T20:54:35,393 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting ad8c182967a345e8ae18d89fcb97ffb1, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731358470813 2024-11-11T20:54:35,393 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting ecdf3824e68d4523ac223c58eb1b517e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731358471500 2024-11-11T20:54:35,393 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d0e2fd88a8e4ef0a8e7af6b203ff51a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731358473711 2024-11-11T20:54:35,401 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc6d436b2ab3693a9c0834e8e24f25cf#C#compaction#299 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:35,401 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/1dda3d0bed36455ab6e543751e71110a is 50, key is test_row_0/C:col10/1731358473711/Put/seqid=0 2024-11-11T20:54:35,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742184_1360 (size=12629) 2024-11-11T20:54:35,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:35,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dc6d436b2ab3693a9c0834e8e24f25cf 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-11T20:54:35,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=A 2024-11-11T20:54:35,561 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:54:35,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=B 2024-11-11T20:54:35,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:54:35,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=C 2024-11-11T20:54:35,562 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:54:35,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119eac5da422224ae1b62dce0d36952a81_dc6d436b2ab3693a9c0834e8e24f25cf is 50, key is test_row_0/A:col10/1731358474916/Put/seqid=0 2024-11-11T20:54:35,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742185_1361 (size=14794) 2024-11-11T20:54:35,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:35,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358535588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:35,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:35,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,594 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:35,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358535589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:35,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:35,599 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:35,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358535594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,600 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:35,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-11T20:54:35,635 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:54:35,636 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:54:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-11T20:54:35,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T20:54:35,637 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:54:35,638 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:54:35,638 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:54:35,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:35,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358535706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:35,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:35,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358535706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:35,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:35,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:35,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358535706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:35,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:35,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T20:54:35,789 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T20:54:35,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:35,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:35,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:35,790 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:35,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:35,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:35,792 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc6d436b2ab3693a9c0834e8e24f25cf#A#compaction#298 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:35,792 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/52a0169fd9ad4cdd89a7fa80fa26adb3 is 175, key is test_row_0/A:col10/1731358473711/Put/seqid=0 2024-11-11T20:54:35,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742186_1362 (size=31583) 2024-11-11T20:54:35,819 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/1dda3d0bed36455ab6e543751e71110a as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/1dda3d0bed36455ab6e543751e71110a 2024-11-11T20:54:35,823 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dc6d436b2ab3693a9c0834e8e24f25cf/C of dc6d436b2ab3693a9c0834e8e24f25cf into 1dda3d0bed36455ab6e543751e71110a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:35,823 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:35,823 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., storeName=dc6d436b2ab3693a9c0834e8e24f25cf/C, priority=13, startTime=1731358475356; duration=0sec 2024-11-11T20:54:35,823 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:35,823 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc6d436b2ab3693a9c0834e8e24f25cf:C 2024-11-11T20:54:35,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:35,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358535916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:35,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,923 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:35,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358535926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:35,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:35,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358535926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:35,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:35,933 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:35,934 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:35,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T20:54:35,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:35,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:35,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:35,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:35,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:35,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T20:54:35,974 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:35,977 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119eac5da422224ae1b62dce0d36952a81_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119eac5da422224ae1b62dce0d36952a81_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:35,978 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/199e87cd0cc649178c85602acd5f6a7c, store: [table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:35,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/199e87cd0cc649178c85602acd5f6a7c is 175, key is test_row_0/A:col10/1731358474916/Put/seqid=0 2024-11-11T20:54:35,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742187_1363 (size=39749) 2024-11-11T20:54:36,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T20:54:36,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:36,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:36,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:36,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:36,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:36,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:36,199 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/52a0169fd9ad4cdd89a7fa80fa26adb3 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/52a0169fd9ad4cdd89a7fa80fa26adb3 2024-11-11T20:54:36,202 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in dc6d436b2ab3693a9c0834e8e24f25cf/A of dc6d436b2ab3693a9c0834e8e24f25cf into 52a0169fd9ad4cdd89a7fa80fa26adb3(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:36,202 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:36,202 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., storeName=dc6d436b2ab3693a9c0834e8e24f25cf/A, priority=13, startTime=1731358475356; duration=0sec 2024-11-11T20:54:36,202 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:36,202 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc6d436b2ab3693a9c0834e8e24f25cf:A 2024-11-11T20:54:36,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:36,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358536225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:36,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:36,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:36,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:36,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:36,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358536236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:36,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:36,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:36,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:36,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T20:54:36,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:36,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:36,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:36,247 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:36,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:36,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:36,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:36,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358536246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:36,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:36,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:36,251 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:36,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T20:54:36,381 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=252, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/199e87cd0cc649178c85602acd5f6a7c 2024-11-11T20:54:36,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/56a545622c934eba8a30c6c31f5415d7 is 50, key is test_row_0/B:col10/1731358474916/Put/seqid=0 2024-11-11T20:54:36,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742188_1364 (size=12151) 2024-11-11T20:54:36,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T20:54:36,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:36,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:36,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:36,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:36,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:36,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:36,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T20:54:36,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:36,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:36,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:36,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:36,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:36,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:36,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T20:54:36,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:36,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:36,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:36,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:36,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:36,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:36,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:36,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358536735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:36,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:36,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:36,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:36,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:36,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358536746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:36,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:36,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:36,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:36,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:36,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358536765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:36,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:36,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:36,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:36,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T20:54:36,795 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/56a545622c934eba8a30c6c31f5415d7 2024-11-11T20:54:36,802 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/9c0e5c8b2afb47efbb3590b42ec629be is 50, key is test_row_0/C:col10/1731358474916/Put/seqid=0 2024-11-11T20:54:36,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742189_1365 (size=12151) 2024-11-11T20:54:36,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T20:54:36,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:36,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:36,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:36,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:36,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:36,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:37,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T20:54:37,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:37,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:37,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:37,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:37,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:37,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:37,165 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T20:54:37,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:37,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:37,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:37,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:37,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:37,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:37,207 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=252 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/9c0e5c8b2afb47efbb3590b42ec629be 2024-11-11T20:54:37,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/199e87cd0cc649178c85602acd5f6a7c as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/199e87cd0cc649178c85602acd5f6a7c 2024-11-11T20:54:37,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/199e87cd0cc649178c85602acd5f6a7c, entries=200, sequenceid=252, filesize=38.8 K 2024-11-11T20:54:37,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/56a545622c934eba8a30c6c31f5415d7 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/56a545622c934eba8a30c6c31f5415d7 2024-11-11T20:54:37,218 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/56a545622c934eba8a30c6c31f5415d7, entries=150, sequenceid=252, filesize=11.9 K 2024-11-11T20:54:37,219 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/9c0e5c8b2afb47efbb3590b42ec629be as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/9c0e5c8b2afb47efbb3590b42ec629be 2024-11-11T20:54:37,222 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/9c0e5c8b2afb47efbb3590b42ec629be, entries=150, sequenceid=252, filesize=11.9 K 2024-11-11T20:54:37,223 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for dc6d436b2ab3693a9c0834e8e24f25cf in 1662ms, sequenceid=252, compaction requested=false 2024-11-11T20:54:37,223 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:37,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T20:54:37,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:37,318 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing dc6d436b2ab3693a9c0834e8e24f25cf 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T20:54:37,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=A 2024-11-11T20:54:37,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:37,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=B 2024-11-11T20:54:37,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:37,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=C 2024-11-11T20:54:37,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:37,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119670aefeb4a1414cbbfa67bbebd03d92_dc6d436b2ab3693a9c0834e8e24f25cf is 50, key is test_row_0/A:col10/1731358475592/Put/seqid=0 2024-11-11T20:54:37,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742190_1366 (size=12454) 2024-11-11T20:54:37,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:37,734 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119670aefeb4a1414cbbfa67bbebd03d92_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119670aefeb4a1414cbbfa67bbebd03d92_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:37,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/c3a0fdcc706f45ec98d95ae42ede902f, store: [table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:37,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/c3a0fdcc706f45ec98d95ae42ede902f is 175, key is test_row_0/A:col10/1731358475592/Put/seqid=0 2024-11-11T20:54:37,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742191_1367 (size=31255) 2024-11-11T20:54:37,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:37,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:37,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T20:54:37,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731358477801, max timestamp=1731358477818Num uniques 859; 2024-11-11T20:54:37,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T20:54:37,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731358477801, max timestamp=1731358477818Num uniques 859; 2024-11-11T20:54:37,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T20:54:37,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731358477801, max timestamp=1731358477818Num uniques 859; 2024-11-11T20:54:37,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T20:54:37,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:37,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358537889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:37,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:37,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:37,902 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:37,903 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:37,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358537901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:37,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:37,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:37,904 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:37,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:37,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358537902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:37,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:37,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:37,914 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:38,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:38,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358538006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:38,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:38,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:38,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:38,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358538006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:38,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:38,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:38,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:38,017 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:38,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:38,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358538015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:38,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:38,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:38,027 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:38,140 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=265, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/c3a0fdcc706f45ec98d95ae42ede902f 2024-11-11T20:54:38,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/77ec3ba16b0f41f58b35372c8c0e74ec is 50, key is test_row_0/B:col10/1731358475592/Put/seqid=0 2024-11-11T20:54:38,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742192_1368 (size=12301) 2024-11-11T20:54:38,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:38,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358538226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:38,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:38,233 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:38,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358538226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:38,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:38,234 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:38,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:38,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:38,234 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:38,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:38,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358538236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:38,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:38,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:38,247 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:38,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:38,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358538535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:38,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:38,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:38,553 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/77ec3ba16b0f41f58b35372c8c0e74ec 2024-11-11T20:54:38,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:38,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358538545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:38,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:38,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:38,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:38,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/f40c3d3b48dc4ad89949684947dfdd42 is 50, key is test_row_0/C:col10/1731358475592/Put/seqid=0 2024-11-11T20:54:38,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742193_1369 (size=12301) 2024-11-11T20:54:38,567 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:38,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358538556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:38,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:38,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:38,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:38,962 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=265 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/f40c3d3b48dc4ad89949684947dfdd42 2024-11-11T20:54:38,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/c3a0fdcc706f45ec98d95ae42ede902f as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/c3a0fdcc706f45ec98d95ae42ede902f 2024-11-11T20:54:38,969 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/c3a0fdcc706f45ec98d95ae42ede902f, entries=150, sequenceid=265, filesize=30.5 K 2024-11-11T20:54:38,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/77ec3ba16b0f41f58b35372c8c0e74ec as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/77ec3ba16b0f41f58b35372c8c0e74ec 2024-11-11T20:54:38,973 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/77ec3ba16b0f41f58b35372c8c0e74ec, entries=150, sequenceid=265, filesize=12.0 K 2024-11-11T20:54:38,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/f40c3d3b48dc4ad89949684947dfdd42 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/f40c3d3b48dc4ad89949684947dfdd42 2024-11-11T20:54:38,977 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/f40c3d3b48dc4ad89949684947dfdd42, entries=150, sequenceid=265, filesize=12.0 K 2024-11-11T20:54:38,977 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for dc6d436b2ab3693a9c0834e8e24f25cf in 1659ms, sequenceid=265, compaction requested=true 2024-11-11T20:54:38,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:38,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:38,978 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-11T20:54:38,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-11T20:54:38,980 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-11T20:54:38,980 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3410 sec 2024-11-11T20:54:38,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 3.3440 sec 2024-11-11T20:54:39,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:39,052 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing dc6d436b2ab3693a9c0834e8e24f25cf 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-11T20:54:39,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=A 2024-11-11T20:54:39,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:54:39,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=B 2024-11-11T20:54:39,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:54:39,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=C 2024-11-11T20:54:39,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:54:39,059 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111920360f6285941938a8326761971bc1d_dc6d436b2ab3693a9c0834e8e24f25cf is 50, key is test_row_0/A:col10/1731358477887/Put/seqid=0 2024-11-11T20:54:39,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742194_1370 (size=14994) 2024-11-11T20:54:39,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358539084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:39,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:39,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:39,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358539085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:39,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:39,087 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:39,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:39,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:39,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:39,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:39,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358539086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:39,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:39,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:39,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:39,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:39,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358539196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:39,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:39,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:39,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:39,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:39,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358539196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:39,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:39,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:39,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:39,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:39,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358539206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:39,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:39,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:39,213 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:39,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358539415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:39,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:39,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:39,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:39,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358539415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:39,426 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:39,426 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:39,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:39,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:39,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358539416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:39,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:39,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:39,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:39,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:39,464 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:39,467 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111920360f6285941938a8326761971bc1d_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111920360f6285941938a8326761971bc1d_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:39,468 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/0364a08ccb9b4190b9a042cf56736bfb, store: [table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:39,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/0364a08ccb9b4190b9a042cf56736bfb is 175, key is test_row_0/A:col10/1731358477887/Put/seqid=0 2024-11-11T20:54:39,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742195_1371 (size=39949) 2024-11-11T20:54:39,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:39,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358539735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:39,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:39,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:39,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:39,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:39,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358539735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:39,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:39,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:39,745 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:39,746 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:39,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358539735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:39,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:39,746 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:39,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T20:54:39,796 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:54:39,797 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:54:39,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-11T20:54:39,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T20:54:39,798 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:54:39,799 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:54:39,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:54:39,823 INFO [Thread-1405 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1405. 2024-11-11T20:54:39,823 DEBUG [Thread-1405 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:54:39,823 DEBUG [Thread-1405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:39,823 DEBUG [Thread-1405 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:39,823 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:54:39,830 INFO [Thread-1409 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1409. 2024-11-11T20:54:39,831 DEBUG [Thread-1409 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:54:39,831 DEBUG [Thread-1409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:39,831 DEBUG [Thread-1409 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:39,831 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:54:39,832 INFO [Thread-1407 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1407. 2024-11-11T20:54:39,832 DEBUG [Thread-1407 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:54:39,832 DEBUG [Thread-1407 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:39,832 DEBUG [Thread-1407 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:39,832 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:54:39,836 INFO [Thread-1413 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1413. 2024-11-11T20:54:39,836 DEBUG [Thread-1413 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:54:39,836 DEBUG [Thread-1413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:39,836 DEBUG [Thread-1413 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:39,836 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:54:39,843 INFO [Thread-1411 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1411. 2024-11-11T20:54:39,843 DEBUG [Thread-1411 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:54:39,843 DEBUG [Thread-1411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:39,843 DEBUG [Thread-1411 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:39,843 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:54:39,872 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=300, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/0364a08ccb9b4190b9a042cf56736bfb 2024-11-11T20:54:39,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/a21e735e39534899a947c82a8050c1ab is 50, key is test_row_0/B:col10/1731358477887/Put/seqid=0 2024-11-11T20:54:39,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742196_1372 (size=12301) 2024-11-11T20:54:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T20:54:39,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T20:54:39,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:39,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:39,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:39,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:39,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:40,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T20:54:40,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:40,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:40,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:40,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:40,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:40,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T20:54:40,257 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:40,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:40,257 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:40,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47586 deadline: 1731358540257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:40,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47570 deadline: 1731358540256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:40,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47580 deadline: 1731358540257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:40,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:40,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:40,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:40,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:40,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:40,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:40,258 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:40,258 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:40,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:54:40,262 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T20:54:40,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:40,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:40,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:40,262 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:40,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:40,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:40,282 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/a21e735e39534899a947c82a8050c1ab 2024-11-11T20:54:40,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/cb2794eb8cab4b3e8562b560db4d0061 is 50, key is test_row_0/C:col10/1731358477887/Put/seqid=0 2024-11-11T20:54:40,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742197_1373 (size=12301) 2024-11-11T20:54:40,415 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T20:54:40,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:40,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:40,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:40,416 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:40,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:40,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:40,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T20:54:40,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T20:54:40,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:40,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:40,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:40,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:40,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:40,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:40,577 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T20:54:40,693 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/cb2794eb8cab4b3e8562b560db4d0061 2024-11-11T20:54:40,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/0364a08ccb9b4190b9a042cf56736bfb as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/0364a08ccb9b4190b9a042cf56736bfb 2024-11-11T20:54:40,706 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/0364a08ccb9b4190b9a042cf56736bfb, entries=200, sequenceid=300, filesize=39.0 K 2024-11-11T20:54:40,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/a21e735e39534899a947c82a8050c1ab as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/a21e735e39534899a947c82a8050c1ab 2024-11-11T20:54:40,710 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/a21e735e39534899a947c82a8050c1ab, entries=150, sequenceid=300, filesize=12.0 K 2024-11-11T20:54:40,711 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/cb2794eb8cab4b3e8562b560db4d0061 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/cb2794eb8cab4b3e8562b560db4d0061 2024-11-11T20:54:40,714 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/cb2794eb8cab4b3e8562b560db4d0061, entries=150, sequenceid=300, filesize=12.0 K 2024-11-11T20:54:40,715 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for dc6d436b2ab3693a9c0834e8e24f25cf in 1663ms, sequenceid=300, compaction requested=true 2024-11-11T20:54:40,715 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:40,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc6d436b2ab3693a9c0834e8e24f25cf:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:54:40,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:40,715 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:54:40,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc6d436b2ab3693a9c0834e8e24f25cf:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:54:40,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:40,715 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:54:40,715 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store dc6d436b2ab3693a9c0834e8e24f25cf:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:54:40,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:54:40,717 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142536 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:54:40,717 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49382 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:54:40,717 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): dc6d436b2ab3693a9c0834e8e24f25cf/B is initiating minor compaction (all files) 2024-11-11T20:54:40,717 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): dc6d436b2ab3693a9c0834e8e24f25cf/A is initiating minor compaction (all files) 2024-11-11T20:54:40,717 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc6d436b2ab3693a9c0834e8e24f25cf/B in TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:40,717 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc6d436b2ab3693a9c0834e8e24f25cf/A in TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:40,717 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/97f998660fad417eb29b86c37ad271ba, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/56a545622c934eba8a30c6c31f5415d7, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/77ec3ba16b0f41f58b35372c8c0e74ec, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/a21e735e39534899a947c82a8050c1ab] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp, totalSize=48.2 K 2024-11-11T20:54:40,717 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/52a0169fd9ad4cdd89a7fa80fa26adb3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/199e87cd0cc649178c85602acd5f6a7c, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/c3a0fdcc706f45ec98d95ae42ede902f, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/0364a08ccb9b4190b9a042cf56736bfb] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp, totalSize=139.2 K 2024-11-11T20:54:40,717 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:40,717 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. files: [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/52a0169fd9ad4cdd89a7fa80fa26adb3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/199e87cd0cc649178c85602acd5f6a7c, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/c3a0fdcc706f45ec98d95ae42ede902f, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/0364a08ccb9b4190b9a042cf56736bfb] 2024-11-11T20:54:40,717 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 97f998660fad417eb29b86c37ad271ba, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731358473711 2024-11-11T20:54:40,717 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52a0169fd9ad4cdd89a7fa80fa26adb3, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731358473711 2024-11-11T20:54:40,717 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 56a545622c934eba8a30c6c31f5415d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731358474916 2024-11-11T20:54:40,718 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 199e87cd0cc649178c85602acd5f6a7c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731358474916 2024-11-11T20:54:40,718 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 77ec3ba16b0f41f58b35372c8c0e74ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731358475572 2024-11-11T20:54:40,718 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3a0fdcc706f45ec98d95ae42ede902f, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731358475572 2024-11-11T20:54:40,718 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting a21e735e39534899a947c82a8050c1ab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731358477887 2024-11-11T20:54:40,718 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0364a08ccb9b4190b9a042cf56736bfb, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731358477887 2024-11-11T20:54:40,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T20:54:40,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:40,726 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing dc6d436b2ab3693a9c0834e8e24f25cf 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T20:54:40,726 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc6d436b2ab3693a9c0834e8e24f25cf#B#compaction#309 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:40,726 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:40,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=A 2024-11-11T20:54:40,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:40,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=B 2024-11-11T20:54:40,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:40,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=C 2024-11-11T20:54:40,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:40,726 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/350579c1cab641fa975f08743928c705 is 50, key is test_row_0/B:col10/1731358477887/Put/seqid=0 2024-11-11T20:54:40,728 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111159619a64825d48c6852e13760b50427d_dc6d436b2ab3693a9c0834e8e24f25cf store=[table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:40,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742198_1374 (size=12915) 2024-11-11T20:54:40,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111de19318689ad4e41a86f644f88ea6df7_dc6d436b2ab3693a9c0834e8e24f25cf is 50, key is test_row_0/A:col10/1731358479083/Put/seqid=0 2024-11-11T20:54:40,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742199_1375 (size=12454) 2024-11-11T20:54:40,751 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111159619a64825d48c6852e13760b50427d_dc6d436b2ab3693a9c0834e8e24f25cf, store=[table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:40,751 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111159619a64825d48c6852e13760b50427d_dc6d436b2ab3693a9c0834e8e24f25cf because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:40,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742200_1376 (size=4469) 2024-11-11T20:54:40,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T20:54:41,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:41,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. as already flushing 2024-11-11T20:54:41,059 INFO [Thread-1398 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1398. 2024-11-11T20:54:41,059 DEBUG [Thread-1398 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:54:41,059 DEBUG [Thread-1398 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:41,059 DEBUG [Thread-1398 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:41,060 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:54:41,096 INFO [Thread-1400 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1400. 2024-11-11T20:54:41,096 DEBUG [Thread-1400 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:54:41,097 DEBUG [Thread-1400 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:41,097 DEBUG [Thread-1400 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:41,097 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:54:41,136 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/350579c1cab641fa975f08743928c705 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/350579c1cab641fa975f08743928c705 2024-11-11T20:54:41,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:41,140 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dc6d436b2ab3693a9c0834e8e24f25cf/B of dc6d436b2ab3693a9c0834e8e24f25cf into 350579c1cab641fa975f08743928c705(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:41,140 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:41,140 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., storeName=dc6d436b2ab3693a9c0834e8e24f25cf/B, priority=12, startTime=1731358480715; duration=0sec 2024-11-11T20:54:41,140 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:54:41,140 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc6d436b2ab3693a9c0834e8e24f25cf:B 2024-11-11T20:54:41,140 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:54:41,141 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49382 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:54:41,141 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): dc6d436b2ab3693a9c0834e8e24f25cf/C is initiating minor compaction (all files) 2024-11-11T20:54:41,141 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of dc6d436b2ab3693a9c0834e8e24f25cf/C in TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:41,141 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/1dda3d0bed36455ab6e543751e71110a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/9c0e5c8b2afb47efbb3590b42ec629be, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/f40c3d3b48dc4ad89949684947dfdd42, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/cb2794eb8cab4b3e8562b560db4d0061] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp, totalSize=48.2 K 2024-11-11T20:54:41,141 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dda3d0bed36455ab6e543751e71110a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1731358473711 2024-11-11T20:54:41,141 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111de19318689ad4e41a86f644f88ea6df7_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111de19318689ad4e41a86f644f88ea6df7_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:41,142 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c0e5c8b2afb47efbb3590b42ec629be, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=252, earliestPutTs=1731358474916 2024-11-11T20:54:41,142 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting f40c3d3b48dc4ad89949684947dfdd42, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=265, earliestPutTs=1731358475572 2024-11-11T20:54:41,142 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting cb2794eb8cab4b3e8562b560db4d0061, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731358477887 2024-11-11T20:54:41,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/ed290a5dc86749bbbe2815f8e8695349, store: [table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:41,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/ed290a5dc86749bbbe2815f8e8695349 is 175, key is test_row_0/A:col10/1731358479083/Put/seqid=0 2024-11-11T20:54:41,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742201_1377 (size=31255) 2024-11-11T20:54:41,146 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=310, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/ed290a5dc86749bbbe2815f8e8695349 2024-11-11T20:54:41,153 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc6d436b2ab3693a9c0834e8e24f25cf#C#compaction#312 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:41,153 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/b36440d946f34e0ebe866b66db0b020d is 50, key is test_row_0/C:col10/1731358477887/Put/seqid=0 2024-11-11T20:54:41,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/5ec34f0707504f12b7dbf3bac6301eab is 50, key is test_row_0/B:col10/1731358479083/Put/seqid=0 2024-11-11T20:54:41,156 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): dc6d436b2ab3693a9c0834e8e24f25cf#A#compaction#310 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:41,157 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/a5112fda12a94e0e95b7488d9c5c289f is 175, key is test_row_0/A:col10/1731358477887/Put/seqid=0 2024-11-11T20:54:41,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742202_1378 (size=12915) 2024-11-11T20:54:41,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742203_1379 (size=12301) 2024-11-11T20:54:41,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742204_1380 (size=31869) 2024-11-11T20:54:41,267 INFO [Thread-1396 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1396. 2024-11-11T20:54:41,267 INFO [Thread-1394 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1394. 2024-11-11T20:54:41,268 DEBUG [Thread-1396 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:54:41,268 DEBUG [Thread-1394 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:54:41,268 DEBUG [Thread-1396 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:41,268 DEBUG [Thread-1394 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:41,268 DEBUG [Thread-1396 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:41,268 DEBUG [Thread-1394 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:41,268 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:54:41,268 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:54:41,277 INFO [Thread-1402 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1402. 2024-11-11T20:54:41,277 DEBUG [Thread-1402 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:54:41,277 DEBUG [Thread-1402 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:41,277 DEBUG [Thread-1402 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:41,277 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:54:41,563 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/5ec34f0707504f12b7dbf3bac6301eab 2024-11-11T20:54:41,569 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/b36440d946f34e0ebe866b66db0b020d as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/b36440d946f34e0ebe866b66db0b020d 2024-11-11T20:54:41,572 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/a5112fda12a94e0e95b7488d9c5c289f as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/a5112fda12a94e0e95b7488d9c5c289f 2024-11-11T20:54:41,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/5a39a54b5ed64d6cb8ada7a3b70020d7 is 50, key is test_row_0/C:col10/1731358479083/Put/seqid=0 2024-11-11T20:54:41,575 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dc6d436b2ab3693a9c0834e8e24f25cf/C of dc6d436b2ab3693a9c0834e8e24f25cf into b36440d946f34e0ebe866b66db0b020d(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:41,575 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:41,575 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., storeName=dc6d436b2ab3693a9c0834e8e24f25cf/C, priority=12, startTime=1731358480715; duration=0sec 2024-11-11T20:54:41,575 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:41,575 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc6d436b2ab3693a9c0834e8e24f25cf:C 2024-11-11T20:54:41,576 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in dc6d436b2ab3693a9c0834e8e24f25cf/A of dc6d436b2ab3693a9c0834e8e24f25cf into a5112fda12a94e0e95b7488d9c5c289f(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:41,576 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:41,576 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., storeName=dc6d436b2ab3693a9c0834e8e24f25cf/A, priority=12, startTime=1731358480715; duration=0sec 2024-11-11T20:54:41,576 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:41,576 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: dc6d436b2ab3693a9c0834e8e24f25cf:A 2024-11-11T20:54:41,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742205_1381 (size=12301) 2024-11-11T20:54:41,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T20:54:41,978 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/5a39a54b5ed64d6cb8ada7a3b70020d7 2024-11-11T20:54:41,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/ed290a5dc86749bbbe2815f8e8695349 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/ed290a5dc86749bbbe2815f8e8695349 2024-11-11T20:54:41,994 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/ed290a5dc86749bbbe2815f8e8695349, entries=150, sequenceid=310, filesize=30.5 K 2024-11-11T20:54:41,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/5ec34f0707504f12b7dbf3bac6301eab as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/5ec34f0707504f12b7dbf3bac6301eab 2024-11-11T20:54:42,000 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/5ec34f0707504f12b7dbf3bac6301eab, entries=150, sequenceid=310, filesize=12.0 K 2024-11-11T20:54:42,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/5a39a54b5ed64d6cb8ada7a3b70020d7 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/5a39a54b5ed64d6cb8ada7a3b70020d7 2024-11-11T20:54:42,005 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/5a39a54b5ed64d6cb8ada7a3b70020d7, entries=150, sequenceid=310, filesize=12.0 K 2024-11-11T20:54:42,006 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=33.54 KB/34350 for dc6d436b2ab3693a9c0834e8e24f25cf in 1280ms, sequenceid=310, compaction requested=false 2024-11-11T20:54:42,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for dc6d436b2ab3693a9c0834e8e24f25cf: 2024-11-11T20:54:42,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:42,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-11T20:54:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-11T20:54:42,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-11T20:54:42,009 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2080 sec 2024-11-11T20:54:42,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.2120 sec 2024-11-11T20:54:43,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T20:54:43,968 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:54:43,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T20:54:43,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-11T20:54:43,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-11T20:54:43,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-11T20:54:43,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-11T20:54:43,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-11T20:54:43,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T20:54:43,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T20:54:43,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1124 2024-11-11T20:54:43,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3371 rows 2024-11-11T20:54:43,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1133 2024-11-11T20:54:43,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3395 rows 2024-11-11T20:54:43,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1131 2024-11-11T20:54:43,968 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3385 rows 2024-11-11T20:54:43,969 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1128 2024-11-11T20:54:43,969 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3382 rows 2024-11-11T20:54:43,969 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1125 2024-11-11T20:54:43,969 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3374 rows 2024-11-11T20:54:43,969 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T20:54:43,969 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:54:43,969 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:43,970 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:43,970 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:54:43,972 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T20:54:43,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T20:54:43,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T20:54:43,978 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358483977"}]},"ts":"1731358483977"} 2024-11-11T20:54:43,979 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T20:54:43,979 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T20:54:43,980 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T20:54:43,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=101, ppid=100, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc6d436b2ab3693a9c0834e8e24f25cf, UNASSIGN}] 2024-11-11T20:54:43,982 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=101, ppid=100, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc6d436b2ab3693a9c0834e8e24f25cf, UNASSIGN 2024-11-11T20:54:43,982 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=101 updating hbase:meta row=dc6d436b2ab3693a9c0834e8e24f25cf, regionState=CLOSING, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:43,984 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=101, ppid=100, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc6d436b2ab3693a9c0834e8e24f25cf, UNASSIGN because future has completed 2024-11-11T20:54:43,984 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T20:54:43,984 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; CloseRegionProcedure dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066}] 2024-11-11T20:54:44,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T20:54:44,137 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] handler.UnassignRegionHandler(122): Close dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:44,137 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T20:54:44,138 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1722): Closing dc6d436b2ab3693a9c0834e8e24f25cf, disabling compactions & flushes 2024-11-11T20:54:44,138 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:44,138 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:44,138 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. after waiting 0 ms 2024-11-11T20:54:44,138 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:44,138 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(2902): Flushing dc6d436b2ab3693a9c0834e8e24f25cf 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T20:54:44,139 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=A 2024-11-11T20:54:44,139 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:44,139 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=B 2024-11-11T20:54:44,139 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:44,139 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK dc6d436b2ab3693a9c0834e8e24f25cf, store=C 2024-11-11T20:54:44,139 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:44,149 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118c515410b50047dda34cc2fea8780148_dc6d436b2ab3693a9c0834e8e24f25cf is 50, key is test_row_1/A:col10/1731358481266/Put/seqid=0 2024-11-11T20:54:44,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742206_1382 (size=9914) 2024-11-11T20:54:44,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T20:54:44,553 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:44,563 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118c515410b50047dda34cc2fea8780148_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118c515410b50047dda34cc2fea8780148_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:44,564 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/d1ad135076484d6098f5e33ad68cea99, store: [table=TestAcidGuarantees family=A region=dc6d436b2ab3693a9c0834e8e24f25cf] 2024-11-11T20:54:44,565 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/d1ad135076484d6098f5e33ad68cea99 is 175, key is test_row_1/A:col10/1731358481266/Put/seqid=0 2024-11-11T20:54:44,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742207_1383 (size=22561) 2024-11-11T20:54:44,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T20:54:44,936 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/188138385396484780189e66800f670f, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/8a53a8b38cd34715a95966edbe2aee7a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/266d5b9cc4a04d10838fea0dee88b65d, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/98c364f002f9417e87c828ecefa17ada, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/40d1b6b19b7046a5bda3571439b2b4b7, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/0b8fd0cb1acb42379547ebbd39ef97f1, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/a166b1910a634cd28606825025e1b64b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/5681fa02631044759a68c1c40b8c009f, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/544b0ecb79d94b1da8b5fc1c5554dae9, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/ce5afa35f2c94867965666813810a1ec, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/22bbf26497e1477ca7cd440778307fe2, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/e383557d462147b88207bd12c13df32a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/6b23ecd846ee442c914e1b65ff0f5d73, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/6f6096e1a86b4bd4b1499bd8c972833c, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/52a0169fd9ad4cdd89a7fa80fa26adb3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/199e87cd0cc649178c85602acd5f6a7c, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/c3a0fdcc706f45ec98d95ae42ede902f, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/0364a08ccb9b4190b9a042cf56736bfb] to archive 2024-11-11T20:54:44,938 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:54:44,940 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/188138385396484780189e66800f670f to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/188138385396484780189e66800f670f 2024-11-11T20:54:44,942 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/8a53a8b38cd34715a95966edbe2aee7a to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/8a53a8b38cd34715a95966edbe2aee7a 2024-11-11T20:54:44,944 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/266d5b9cc4a04d10838fea0dee88b65d to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/266d5b9cc4a04d10838fea0dee88b65d 2024-11-11T20:54:44,945 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/98c364f002f9417e87c828ecefa17ada to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/98c364f002f9417e87c828ecefa17ada 2024-11-11T20:54:44,946 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/40d1b6b19b7046a5bda3571439b2b4b7 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/40d1b6b19b7046a5bda3571439b2b4b7 2024-11-11T20:54:44,948 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/0b8fd0cb1acb42379547ebbd39ef97f1 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/0b8fd0cb1acb42379547ebbd39ef97f1 2024-11-11T20:54:44,949 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/a166b1910a634cd28606825025e1b64b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/a166b1910a634cd28606825025e1b64b 2024-11-11T20:54:44,950 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/5681fa02631044759a68c1c40b8c009f to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/5681fa02631044759a68c1c40b8c009f 2024-11-11T20:54:44,952 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/544b0ecb79d94b1da8b5fc1c5554dae9 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/544b0ecb79d94b1da8b5fc1c5554dae9 2024-11-11T20:54:44,953 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/ce5afa35f2c94867965666813810a1ec to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/ce5afa35f2c94867965666813810a1ec 2024-11-11T20:54:44,954 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/22bbf26497e1477ca7cd440778307fe2 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/22bbf26497e1477ca7cd440778307fe2 2024-11-11T20:54:44,955 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/e383557d462147b88207bd12c13df32a to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/e383557d462147b88207bd12c13df32a 2024-11-11T20:54:44,956 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/6b23ecd846ee442c914e1b65ff0f5d73 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/6b23ecd846ee442c914e1b65ff0f5d73 2024-11-11T20:54:44,958 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/6f6096e1a86b4bd4b1499bd8c972833c to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/6f6096e1a86b4bd4b1499bd8c972833c 2024-11-11T20:54:44,959 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/52a0169fd9ad4cdd89a7fa80fa26adb3 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/52a0169fd9ad4cdd89a7fa80fa26adb3 2024-11-11T20:54:44,960 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/199e87cd0cc649178c85602acd5f6a7c to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/199e87cd0cc649178c85602acd5f6a7c 2024-11-11T20:54:44,961 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/c3a0fdcc706f45ec98d95ae42ede902f to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/c3a0fdcc706f45ec98d95ae42ede902f 2024-11-11T20:54:44,962 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/0364a08ccb9b4190b9a042cf56736bfb to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/0364a08ccb9b4190b9a042cf56736bfb 2024-11-11T20:54:44,966 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/bca58eb419764460afef7595bde276a5, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/7b723a1064394268b4bf6b890d4514f3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/81e835a8a6a5491f853a7136a746da64, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/df8e43f1d1944d36805c23a3d4c32e68, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/61172cbbf9e24987af07bed4ad8fbaaa, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/64aacecff2c144f3bc5e5353e8ffc89f, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/7c55d131c3d6467299ec6fe65f4364f1, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/a1c83f8b5a0946f1b0f4d15cad66e0a8, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/ffbad171e8c04b17923d97d119a4f20a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/110a1c95728441c2a3ca70da4f7f03be, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/e0666c896d9a47d381e3cee283d1500a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/269d3578f0744da4a2a1f2d0036b2c3b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/9d8fc233e80b4a548903a1d97b06321e, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/97f998660fad417eb29b86c37ad271ba, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/f575aee3bf024717a6f5c0de7903bb80, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/56a545622c934eba8a30c6c31f5415d7, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/77ec3ba16b0f41f58b35372c8c0e74ec, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/a21e735e39534899a947c82a8050c1ab] to archive 2024-11-11T20:54:44,967 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:54:44,968 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/bca58eb419764460afef7595bde276a5 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/bca58eb419764460afef7595bde276a5 2024-11-11T20:54:44,969 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=321, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/d1ad135076484d6098f5e33ad68cea99 2024-11-11T20:54:44,969 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/7b723a1064394268b4bf6b890d4514f3 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/7b723a1064394268b4bf6b890d4514f3 2024-11-11T20:54:44,970 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/81e835a8a6a5491f853a7136a746da64 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/81e835a8a6a5491f853a7136a746da64 2024-11-11T20:54:44,971 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/df8e43f1d1944d36805c23a3d4c32e68 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/df8e43f1d1944d36805c23a3d4c32e68 2024-11-11T20:54:44,972 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/61172cbbf9e24987af07bed4ad8fbaaa to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/61172cbbf9e24987af07bed4ad8fbaaa 2024-11-11T20:54:44,973 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/64aacecff2c144f3bc5e5353e8ffc89f to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/64aacecff2c144f3bc5e5353e8ffc89f 2024-11-11T20:54:44,974 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/7c55d131c3d6467299ec6fe65f4364f1 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/7c55d131c3d6467299ec6fe65f4364f1 2024-11-11T20:54:44,975 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/2bd223e9a4484ec7a8b66d38d84b2f2d is 50, key is test_row_1/B:col10/1731358481266/Put/seqid=0 2024-11-11T20:54:44,975 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/a1c83f8b5a0946f1b0f4d15cad66e0a8 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/a1c83f8b5a0946f1b0f4d15cad66e0a8 2024-11-11T20:54:44,976 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/ffbad171e8c04b17923d97d119a4f20a to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/ffbad171e8c04b17923d97d119a4f20a 2024-11-11T20:54:44,977 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/110a1c95728441c2a3ca70da4f7f03be to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/110a1c95728441c2a3ca70da4f7f03be 2024-11-11T20:54:44,977 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/e0666c896d9a47d381e3cee283d1500a to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/e0666c896d9a47d381e3cee283d1500a 2024-11-11T20:54:44,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742208_1384 (size=9857) 2024-11-11T20:54:44,978 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/269d3578f0744da4a2a1f2d0036b2c3b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/269d3578f0744da4a2a1f2d0036b2c3b 2024-11-11T20:54:44,979 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/9d8fc233e80b4a548903a1d97b06321e to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/9d8fc233e80b4a548903a1d97b06321e 2024-11-11T20:54:44,979 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/97f998660fad417eb29b86c37ad271ba to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/97f998660fad417eb29b86c37ad271ba 2024-11-11T20:54:44,980 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/f575aee3bf024717a6f5c0de7903bb80 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/f575aee3bf024717a6f5c0de7903bb80 2024-11-11T20:54:44,981 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/56a545622c934eba8a30c6c31f5415d7 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/56a545622c934eba8a30c6c31f5415d7 2024-11-11T20:54:44,982 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/77ec3ba16b0f41f58b35372c8c0e74ec to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/77ec3ba16b0f41f58b35372c8c0e74ec 2024-11-11T20:54:44,983 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/a21e735e39534899a947c82a8050c1ab to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/a21e735e39534899a947c82a8050c1ab 2024-11-11T20:54:44,985 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/c7492faa43944475a8650e334f2b6f6b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/15aeb2171f63446990ba28a70e582a26, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/82ee1638a9044e02b90cf79a0d8140ae, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/cd1df2c0a49a49d0960ab715ef285cc3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/3fac554d9a034750a1c16cee0d812434, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/ada68df1a6d346f09e627c4bbb51e253, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/ff60676ded3a4de184888b5aea5df04a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/d3e1933f3e2e4dc08721f11dc089b253, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/4e396caddf5741798a6d240c8f331d0d, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/95e7120748354e95b89e2acb07456790, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/ad8c182967a345e8ae18d89fcb97ffb1, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/b97dc3d2f701441c9c0c96852625ee0d, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/ecdf3824e68d4523ac223c58eb1b517e, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/1dda3d0bed36455ab6e543751e71110a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/6d0e2fd88a8e4ef0a8e7af6b203ff51a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/9c0e5c8b2afb47efbb3590b42ec629be, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/f40c3d3b48dc4ad89949684947dfdd42, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/cb2794eb8cab4b3e8562b560db4d0061] to archive 2024-11-11T20:54:44,985 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:54:44,986 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/c7492faa43944475a8650e334f2b6f6b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/c7492faa43944475a8650e334f2b6f6b 2024-11-11T20:54:44,987 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/15aeb2171f63446990ba28a70e582a26 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/15aeb2171f63446990ba28a70e582a26 2024-11-11T20:54:44,988 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/82ee1638a9044e02b90cf79a0d8140ae to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/82ee1638a9044e02b90cf79a0d8140ae 2024-11-11T20:54:44,989 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/cd1df2c0a49a49d0960ab715ef285cc3 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/cd1df2c0a49a49d0960ab715ef285cc3 2024-11-11T20:54:44,990 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/3fac554d9a034750a1c16cee0d812434 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/3fac554d9a034750a1c16cee0d812434 2024-11-11T20:54:44,991 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/ada68df1a6d346f09e627c4bbb51e253 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/ada68df1a6d346f09e627c4bbb51e253 2024-11-11T20:54:44,991 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/ff60676ded3a4de184888b5aea5df04a to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/ff60676ded3a4de184888b5aea5df04a 2024-11-11T20:54:44,992 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/d3e1933f3e2e4dc08721f11dc089b253 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/d3e1933f3e2e4dc08721f11dc089b253 2024-11-11T20:54:44,993 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/4e396caddf5741798a6d240c8f331d0d to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/4e396caddf5741798a6d240c8f331d0d 2024-11-11T20:54:44,994 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/95e7120748354e95b89e2acb07456790 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/95e7120748354e95b89e2acb07456790 2024-11-11T20:54:44,995 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/ad8c182967a345e8ae18d89fcb97ffb1 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/ad8c182967a345e8ae18d89fcb97ffb1 2024-11-11T20:54:44,996 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/b97dc3d2f701441c9c0c96852625ee0d to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/b97dc3d2f701441c9c0c96852625ee0d 2024-11-11T20:54:44,996 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/ecdf3824e68d4523ac223c58eb1b517e to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/ecdf3824e68d4523ac223c58eb1b517e 2024-11-11T20:54:44,997 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/1dda3d0bed36455ab6e543751e71110a to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/1dda3d0bed36455ab6e543751e71110a 2024-11-11T20:54:44,998 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/6d0e2fd88a8e4ef0a8e7af6b203ff51a to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/6d0e2fd88a8e4ef0a8e7af6b203ff51a 2024-11-11T20:54:44,999 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/9c0e5c8b2afb47efbb3590b42ec629be to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/9c0e5c8b2afb47efbb3590b42ec629be 2024-11-11T20:54:45,000 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/f40c3d3b48dc4ad89949684947dfdd42 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/f40c3d3b48dc4ad89949684947dfdd42 2024-11-11T20:54:45,001 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/cb2794eb8cab4b3e8562b560db4d0061 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/cb2794eb8cab4b3e8562b560db4d0061 2024-11-11T20:54:45,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T20:54:45,380 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/2bd223e9a4484ec7a8b66d38d84b2f2d 2024-11-11T20:54:45,393 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/a52d35f0a97a48da985c4034ddde9a97 is 50, key is test_row_1/C:col10/1731358481266/Put/seqid=0 2024-11-11T20:54:45,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742209_1385 (size=9857) 2024-11-11T20:54:45,800 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=321 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/a52d35f0a97a48da985c4034ddde9a97 2024-11-11T20:54:45,811 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/A/d1ad135076484d6098f5e33ad68cea99 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/d1ad135076484d6098f5e33ad68cea99 2024-11-11T20:54:45,816 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/d1ad135076484d6098f5e33ad68cea99, entries=100, sequenceid=321, filesize=22.0 K 2024-11-11T20:54:45,817 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/B/2bd223e9a4484ec7a8b66d38d84b2f2d as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/2bd223e9a4484ec7a8b66d38d84b2f2d 2024-11-11T20:54:45,819 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/2bd223e9a4484ec7a8b66d38d84b2f2d, entries=100, sequenceid=321, filesize=9.6 K 2024-11-11T20:54:45,820 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/.tmp/C/a52d35f0a97a48da985c4034ddde9a97 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/a52d35f0a97a48da985c4034ddde9a97 2024-11-11T20:54:45,823 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/a52d35f0a97a48da985c4034ddde9a97, entries=100, sequenceid=321, filesize=9.6 K 2024-11-11T20:54:45,823 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for dc6d436b2ab3693a9c0834e8e24f25cf in 1685ms, sequenceid=321, compaction requested=true 2024-11-11T20:54:45,827 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/recovered.edits/324.seqid, newMaxSeqId=324, maxSeqId=4 2024-11-11T20:54:45,828 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. 2024-11-11T20:54:45,828 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] regionserver.HRegion(1676): Region close journal for dc6d436b2ab3693a9c0834e8e24f25cf: Waiting for close lock at 1731358484137Running coprocessor pre-close hooks at 1731358484137Disabling compacts and flushes for region at 1731358484137Disabling writes for close at 1731358484138 (+1 ms)Obtaining lock to block concurrent updates at 1731358484138Preparing flush snapshotting stores in dc6d436b2ab3693a9c0834e8e24f25cf at 1731358484138Finished memstore snapshotting TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731358484139 (+1 ms)Flushing stores of TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf. at 1731358484141 (+2 ms)Flushing dc6d436b2ab3693a9c0834e8e24f25cf/A: creating writer at 1731358484141Flushing mob file dc6d436b2ab3693a9c0834e8e24f25cf/A: appending metadata at 1731358484148 (+7 ms)Flushing mob file dc6d436b2ab3693a9c0834e8e24f25cf/A: closing flushed file at 1731358484148Flushing dc6d436b2ab3693a9c0834e8e24f25cf/A: appending metadata at 1731358484564 (+416 ms)Flushing dc6d436b2ab3693a9c0834e8e24f25cf/A: closing flushed file at 1731358484564Flushing dc6d436b2ab3693a9c0834e8e24f25cf/B: creating writer at 1731358484972 (+408 ms)Flushing dc6d436b2ab3693a9c0834e8e24f25cf/B: appending metadata at 1731358484974 (+2 ms)Flushing dc6d436b2ab3693a9c0834e8e24f25cf/B: closing flushed file at 1731358484975 (+1 ms)Flushing dc6d436b2ab3693a9c0834e8e24f25cf/C: creating writer at 1731358485388 (+413 ms)Flushing dc6d436b2ab3693a9c0834e8e24f25cf/C: appending metadata at 1731358485392 (+4 ms)Flushing dc6d436b2ab3693a9c0834e8e24f25cf/C: closing flushed file at 1731358485393 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17c409a6: reopening flushed file at 1731358485810 (+417 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@227e23cd: reopening flushed file at 1731358485816 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f0bfe40: reopening flushed file at 1731358485820 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for dc6d436b2ab3693a9c0834e8e24f25cf in 1685ms, sequenceid=321, compaction requested=true at 1731358485823 (+3 ms)Writing region close event to WAL at 1731358485825 (+2 ms)Running coprocessor post-close hooks at 1731358485828 (+3 ms)Closed at 1731358485828 2024-11-11T20:54:45,829 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=102}] handler.UnassignRegionHandler(157): Closed dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:45,830 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=101 updating hbase:meta row=dc6d436b2ab3693a9c0834e8e24f25cf, regionState=CLOSED 2024-11-11T20:54:45,831 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=102, ppid=101, state=RUNNABLE, hasLock=false; CloseRegionProcedure dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 because future has completed 2024-11-11T20:54:45,833 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-11T20:54:45,834 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; CloseRegionProcedure dc6d436b2ab3693a9c0834e8e24f25cf, server=51ca66f7ee3c,44669,1731358362066 in 1.8480 sec 2024-11-11T20:54:45,835 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=101, resume processing ppid=100 2024-11-11T20:54:45,835 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, ppid=100, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=dc6d436b2ab3693a9c0834e8e24f25cf, UNASSIGN in 1.8530 sec 2024-11-11T20:54:45,837 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-11T20:54:45,837 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8550 sec 2024-11-11T20:54:45,838 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358485838"}]},"ts":"1731358485838"} 2024-11-11T20:54:45,839 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T20:54:45,839 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T20:54:45,840 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8670 sec 2024-11-11T20:54:46,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T20:54:46,126 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:54:46,127 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T20:54:46,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:54:46,130 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=103, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:54:46,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T20:54:46,131 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=103, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:54:46,134 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:46,139 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A, FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B, FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C, FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/recovered.edits] 2024-11-11T20:54:46,144 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/a5112fda12a94e0e95b7488d9c5c289f to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/a5112fda12a94e0e95b7488d9c5c289f 2024-11-11T20:54:46,147 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/d1ad135076484d6098f5e33ad68cea99 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/d1ad135076484d6098f5e33ad68cea99 2024-11-11T20:54:46,149 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/ed290a5dc86749bbbe2815f8e8695349 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/A/ed290a5dc86749bbbe2815f8e8695349 2024-11-11T20:54:46,153 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/2bd223e9a4484ec7a8b66d38d84b2f2d to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/2bd223e9a4484ec7a8b66d38d84b2f2d 2024-11-11T20:54:46,155 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/350579c1cab641fa975f08743928c705 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/350579c1cab641fa975f08743928c705 2024-11-11T20:54:46,157 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/5ec34f0707504f12b7dbf3bac6301eab to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/B/5ec34f0707504f12b7dbf3bac6301eab 2024-11-11T20:54:46,161 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/5a39a54b5ed64d6cb8ada7a3b70020d7 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/5a39a54b5ed64d6cb8ada7a3b70020d7 2024-11-11T20:54:46,163 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/a52d35f0a97a48da985c4034ddde9a97 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/a52d35f0a97a48da985c4034ddde9a97 2024-11-11T20:54:46,165 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/b36440d946f34e0ebe866b66db0b020d to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/C/b36440d946f34e0ebe866b66db0b020d 2024-11-11T20:54:46,170 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/recovered.edits/324.seqid to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf/recovered.edits/324.seqid 2024-11-11T20:54:46,171 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:46,171 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T20:54:46,172 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T20:54:46,173 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T20:54:46,178 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110a6a9b5d9f114d219cc57352a6c8a1b9_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110a6a9b5d9f114d219cc57352a6c8a1b9_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:46,179 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112117c6f0853d4b39a7411853f04e5123_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112117c6f0853d4b39a7411853f04e5123_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:46,181 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115c18b26b65be47ecbaafea28f5921fd4_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115c18b26b65be47ecbaafea28f5921fd4_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:46,183 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117f1dbf16a05f41cd9385172bdf1c1783_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117f1dbf16a05f41cd9385172bdf1c1783_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:46,184 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118c515410b50047dda34cc2fea8780148_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118c515410b50047dda34cc2fea8780148_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:46,186 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111920360f6285941938a8326761971bc1d_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111920360f6285941938a8326761971bc1d_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:46,188 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119670aefeb4a1414cbbfa67bbebd03d92_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119670aefeb4a1414cbbfa67bbebd03d92_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:46,189 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119cb2c34196184b4ebe40cb9db5ec214c_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119cb2c34196184b4ebe40cb9db5ec214c_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:46,190 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119eac5da422224ae1b62dce0d36952a81_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119eac5da422224ae1b62dce0d36952a81_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:46,191 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a23d76584770430cb73ff30a021d81ad_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a23d76584770430cb73ff30a021d81ad_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:46,192 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a2bbfd3a950142b0a635883e5f0f0128_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a2bbfd3a950142b0a635883e5f0f0128_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:46,192 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c11d62cc346e4d748a85d16179a660fd_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111c11d62cc346e4d748a85d16179a660fd_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:46,193 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d341b64bb0944de5af1deeefe3cf823c_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d341b64bb0944de5af1deeefe3cf823c_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:46,194 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111de19318689ad4e41a86f644f88ea6df7_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111de19318689ad4e41a86f644f88ea6df7_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:46,195 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e7369472e7bf488e8a356db26099c149_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e7369472e7bf488e8a356db26099c149_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:46,196 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f92c9c6a6dd046f19d2fd9166bfacdb5_dc6d436b2ab3693a9c0834e8e24f25cf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f92c9c6a6dd046f19d2fd9166bfacdb5_dc6d436b2ab3693a9c0834e8e24f25cf 2024-11-11T20:54:46,196 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T20:54:46,197 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=103, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:54:46,199 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T20:54:46,201 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T20:54:46,202 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=103, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:54:46,202 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T20:54:46,203 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731358486202"}]},"ts":"9223372036854775807"} 2024-11-11T20:54:46,204 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T20:54:46,204 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => dc6d436b2ab3693a9c0834e8e24f25cf, NAME => 'TestAcidGuarantees,,1731358456683.dc6d436b2ab3693a9c0834e8e24f25cf.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T20:54:46,204 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T20:54:46,205 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731358486204"}]},"ts":"9223372036854775807"} 2024-11-11T20:54:46,206 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T20:54:46,207 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=103, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:54:46,208 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 80 msec 2024-11-11T20:54:46,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T20:54:46,236 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T20:54:46,236 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:54:46,252 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=252 (was 246) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2048145703_22 at /127.0.0.1:58194 [Waiting for operation #122] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2048145703_22 at /127.0.0.1:58170 [Waiting for operation #1612] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/51ca66f7ee3c:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-634654397_22 at /127.0.0.1:58172 [Waiting for operation #1566] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-10 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-634654397_22 at /127.0.0.1:55866 [Waiting for operation #1668] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=308 (was 350), ProcessCount=11 (was 11), AvailableMemoryMB=4029 (was 4076) 2024-11-11T20:54:46,263 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=252, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=308, ProcessCount=11, AvailableMemoryMB=4028 2024-11-11T20:54:46,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T20:54:46,265 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T20:54:46,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=104, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T20:54:46,267 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T20:54:46,267 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:46,267 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 104 2024-11-11T20:54:46,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-11T20:54:46,268 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T20:54:46,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742210_1386 (size=1061) 2024-11-11T20:54:46,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-11T20:54:46,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-11T20:54:46,680 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23 2024-11-11T20:54:46,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742211_1387 (size=53) 2024-11-11T20:54:46,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-11T20:54:47,091 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:54:47,092 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 82ee51cab8163979e03e6fa77fa08aaf, disabling compactions & flushes 2024-11-11T20:54:47,092 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:47,092 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:47,092 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. after waiting 0 ms 2024-11-11T20:54:47,092 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:47,092 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:47,092 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 82ee51cab8163979e03e6fa77fa08aaf: Waiting for close lock at 1731358487092Disabling compacts and flushes for region at 1731358487092Disabling writes for close at 1731358487092Writing region close event to WAL at 1731358487092Closed at 1731358487092 2024-11-11T20:54:47,095 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T20:54:47,095 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731358487095"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358487095"}]},"ts":"1731358487095"} 2024-11-11T20:54:47,098 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T20:54:47,100 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T20:54:47,101 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358487100"}]},"ts":"1731358487100"} 2024-11-11T20:54:47,102 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T20:54:47,103 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=82ee51cab8163979e03e6fa77fa08aaf, ASSIGN}] 2024-11-11T20:54:47,104 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=82ee51cab8163979e03e6fa77fa08aaf, ASSIGN 2024-11-11T20:54:47,105 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=82ee51cab8163979e03e6fa77fa08aaf, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,44669,1731358362066; forceNewPlan=false, retain=false 2024-11-11T20:54:47,256 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=82ee51cab8163979e03e6fa77fa08aaf, regionState=OPENING, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:47,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=105, ppid=104, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=82ee51cab8163979e03e6fa77fa08aaf, ASSIGN because future has completed 2024-11-11T20:54:47,260 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066}] 2024-11-11T20:54:47,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-11T20:54:47,420 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:47,420 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7752): Opening region: {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:54:47,421 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:54:47,421 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:54:47,422 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7794): checking encryption for 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:54:47,422 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(7797): checking classloading for 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:54:47,424 INFO [StoreOpener-82ee51cab8163979e03e6fa77fa08aaf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:54:47,425 INFO [StoreOpener-82ee51cab8163979e03e6fa77fa08aaf-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T20:54:47,426 INFO [StoreOpener-82ee51cab8163979e03e6fa77fa08aaf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82ee51cab8163979e03e6fa77fa08aaf columnFamilyName A 2024-11-11T20:54:47,426 DEBUG [StoreOpener-82ee51cab8163979e03e6fa77fa08aaf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:47,426 INFO [StoreOpener-82ee51cab8163979e03e6fa77fa08aaf-1 {}] regionserver.HStore(327): Store=82ee51cab8163979e03e6fa77fa08aaf/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:54:47,427 INFO [StoreOpener-82ee51cab8163979e03e6fa77fa08aaf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:54:47,428 INFO [StoreOpener-82ee51cab8163979e03e6fa77fa08aaf-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T20:54:47,428 INFO [StoreOpener-82ee51cab8163979e03e6fa77fa08aaf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82ee51cab8163979e03e6fa77fa08aaf columnFamilyName B 2024-11-11T20:54:47,428 DEBUG [StoreOpener-82ee51cab8163979e03e6fa77fa08aaf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:47,429 INFO [StoreOpener-82ee51cab8163979e03e6fa77fa08aaf-1 {}] regionserver.HStore(327): Store=82ee51cab8163979e03e6fa77fa08aaf/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:54:47,429 INFO [StoreOpener-82ee51cab8163979e03e6fa77fa08aaf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:54:47,430 INFO [StoreOpener-82ee51cab8163979e03e6fa77fa08aaf-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T20:54:47,431 INFO [StoreOpener-82ee51cab8163979e03e6fa77fa08aaf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 82ee51cab8163979e03e6fa77fa08aaf columnFamilyName C 2024-11-11T20:54:47,431 DEBUG [StoreOpener-82ee51cab8163979e03e6fa77fa08aaf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:54:47,431 INFO [StoreOpener-82ee51cab8163979e03e6fa77fa08aaf-1 {}] regionserver.HStore(327): Store=82ee51cab8163979e03e6fa77fa08aaf/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:54:47,432 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:47,432 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1038): replaying wal for 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:54:47,432 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:54:47,433 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:54:47,434 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1048): stopping wal replay for 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:54:47,434 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1060): Cleaning up temporary data for 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:54:47,435 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T20:54:47,437 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1093): writing seq id for 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:54:47,439 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:54:47,440 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1114): Opened 82ee51cab8163979e03e6fa77fa08aaf; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59106408, jitterRate=-0.11924588680267334}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T20:54:47,440 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:54:47,441 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegion(1006): Region open journal for 82ee51cab8163979e03e6fa77fa08aaf: Running coprocessor pre-open hook at 1731358487422Writing region info on filesystem at 1731358487422Initializing all the Stores at 1731358487423 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358487423Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358487423Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358487423Cleaning up temporary data from old regions at 1731358487434 (+11 ms)Running coprocessor post-open hooks at 1731358487440 (+6 ms)Region opened successfully at 1731358487441 (+1 ms) 2024-11-11T20:54:47,442 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., pid=106, masterSystemTime=1731358487413 2024-11-11T20:54:47,445 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:47,445 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=106}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:47,446 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=105 updating hbase:meta row=82ee51cab8163979e03e6fa77fa08aaf, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:47,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=106, ppid=105, state=RUNNABLE, hasLock=false; OpenRegionProcedure 82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 because future has completed 2024-11-11T20:54:47,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-11T20:54:47,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; OpenRegionProcedure 82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 in 189 msec 2024-11-11T20:54:47,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-11-11T20:54:47,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=82ee51cab8163979e03e6fa77fa08aaf, ASSIGN in 348 msec 2024-11-11T20:54:47,453 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T20:54:47,454 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358487453"}]},"ts":"1731358487453"} 2024-11-11T20:54:47,455 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T20:54:47,456 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=104, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T20:54:47,457 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1910 sec 2024-11-11T20:54:48,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-11-11T20:54:48,417 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:54:48,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6516064e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,420 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:54:48,421 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:54:48,422 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:54:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:54:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:54:48,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2550c903, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:54:48,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:54:48,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:48,425 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:54:48,426 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2abba96e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,426 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:54:48,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:54:48,428 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:54:48,429 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36318, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:54:48,430 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T20:54:48,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 51ca66f7ee3c,38429,1731358361462 2024-11-11T20:54:48,431 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6a7c9010 2024-11-11T20:54:48,432 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T20:54:48,433 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52134, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T20:54:48,435 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59de117f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,435 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:54:48,435 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:54:48,436 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:54:48,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:54:48,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:54:48,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c722924, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,436 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:54:48,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:54:48,437 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:48,438 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:54:48,438 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@771765fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,439 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@472a8168, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,439 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:54:48,439 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:54:48,440 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:54:48,440 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:54:48,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:54:48,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@740c0c1e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:54:48,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:54:48,441 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:48,442 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:54:48,443 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8eb4104, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,444 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f010827, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,444 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:54:48,444 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:54:48,444 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:54:48,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:54:48,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:54:48,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39cf9fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:54:48,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:54:48,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:48,446 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52190, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:54:48,447 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58a2ae96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,447 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2822c33e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,447 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:54:48,448 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:54:48,448 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:54:48,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:54:48,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:54:48,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e61b9a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:54:48,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:54:48,449 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:48,450 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52220, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:54:48,450 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c0b5b50, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,451 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f59c6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,451 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:54:48,451 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:54:48,452 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:54:48,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:54:48,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:54:48,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d0a6877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:54:48,452 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:54:48,453 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:48,454 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52238, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:54:48,454 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d9cd885, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42daa6ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,455 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:54:48,455 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:54:48,456 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:54:48,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:54:48,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:54:48,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67918376, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,456 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:54:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:54:48,457 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:48,458 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:54:48,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@428af9a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,459 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@366fb241, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,459 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:54:48,459 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:54:48,460 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:54:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:54:48,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:54:48,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b6bce9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:54:48,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:54:48,461 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:48,462 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52268, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:54:48,463 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55b12ad1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45e8e518, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,464 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:54:48,464 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:54:48,465 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:54:48,465 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:54:48,465 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:54:48,465 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fdabe61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,465 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:54:48,465 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:54:48,466 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:48,466 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:54:48,467 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4db81065, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,468 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10ff7be5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,468 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:54:48,468 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:54:48,469 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:54:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:54:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:54:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41a40bda, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:54:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:54:48,469 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:48,470 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:54:48,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a56e311, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,471 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55c1b90e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,471 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:54:48,471 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:54:48,471 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:54:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:54:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:54:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12f2e147, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,471 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:54:48,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:54:48,472 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:54:48,472 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52328, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:54:48,473 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ad254b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:54:48,473 DEBUG [Thread-1767 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:54:48,473 DEBUG [Thread-1775 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:54:48,473 DEBUG [Thread-1769 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:54:48,473 DEBUG [Thread-1784 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:54:48,474 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:54:48,474 DEBUG [Thread-1782 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:54:48,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:54:48,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:54:48,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:54:48,474 DEBUG [Thread-1773 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:54:48,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:54:48,474 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:54:48,474 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:54:48,474 DEBUG [Thread-1771 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:54:48,474 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:54:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:54:48,475 DEBUG [Thread-1778 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:54:48,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-11T20:54:48,475 DEBUG [Thread-1780 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:54:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:54:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:54:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:54:48,475 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:54:48,475 DEBUG [Thread-1786 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:54:48,480 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:54:48,480 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36354, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:54:48,480 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:54:48,480 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36342, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:54:48,480 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36332, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:54:48,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:54:48,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:54:48,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:54:48,481 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:54:48,481 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:54:48,481 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:54:48,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T20:54:48,482 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36392, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:54:48,482 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36368, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:54:48,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:54:48,482 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:54:48,482 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:54:48,482 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:54:48,482 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:54:48,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:54:48,483 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:54:48,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:54:48,485 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:54:48,486 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:54:48,487 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:54:48,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:54:48,488 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36366, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:54:48,489 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36408, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:54:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:54:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:54:48,491 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:54:48,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2] 2024-11-11T20:54:48,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82ee51cab8163979e03e6fa77fa08aaf 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T20:54:48,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=A 2024-11-11T20:54:48,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:48,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=B 2024-11-11T20:54:48,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:48,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=C 2024-11-11T20:54:48,492 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:48,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:54:48,517 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/3d8380b8e9bd4f7c97b112eb4beef50d is 50, key is test_row_0/A:col10/1731358488487/Put/seqid=0 2024-11-11T20:54:48,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742212_1388 (size=12001) 2024-11-11T20:54:48,539 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/3d8380b8e9bd4f7c97b112eb4beef50d 2024-11-11T20:54:48,543 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358488514, max timestamp=1731358488540Num uniques 859; 2024-11-11T20:54:48,543 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358488514, max timestamp=1731358488540Num uniques 859; 2024-11-11T20:54:48,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:54:48,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:54:48,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731358488514, max timestamp=1731358488540Num uniques 859; 2024-11-11T20:54:48,544 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T20:54:48,571 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/4807f26a508e420bb4a6da9eaeb048fc is 50, key is test_row_0/B:col10/1731358488487/Put/seqid=0 2024-11-11T20:54:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T20:54:48,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742213_1389 (size=12001) 2024-11-11T20:54:48,587 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/4807f26a508e420bb4a6da9eaeb048fc 2024-11-11T20:54:48,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:48,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358548579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:48,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,592 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:48,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:48,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358548580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:48,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1731358548583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:48,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:48,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358548584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:48,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:48,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358548586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:48,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,601 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:48,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:48,619 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/5b53510121a44ce08409a48bdd04ffba is 50, key is test_row_0/C:col10/1731358488487/Put/seqid=0 2024-11-11T20:54:48,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742214_1390 (size=12001) 2024-11-11T20:54:48,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T20:54:48,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:48,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:48,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:48,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:48,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:48,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:48,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:48,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358548696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,700 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:48,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:48,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358548705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:48,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1731358548705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:48,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358548705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,711 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:48,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358548705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:48,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,712 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:48,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T20:54:48,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:48,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:48,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:48,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:48,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:48,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:48,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T20:54:48,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:48,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358548905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:48,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,909 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:48,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:48,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358548916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:48,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:48,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:48,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358548916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:48,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:48,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:48,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358548916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:48,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:48,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,925 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:48,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1731358548916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:48,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:48,926 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:48,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T20:54:48,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:48,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:48,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:48,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:48,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:48,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:49,025 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/5b53510121a44ce08409a48bdd04ffba 2024-11-11T20:54:49,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/3d8380b8e9bd4f7c97b112eb4beef50d as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/3d8380b8e9bd4f7c97b112eb4beef50d 2024-11-11T20:54:49,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/3d8380b8e9bd4f7c97b112eb4beef50d, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T20:54:49,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/4807f26a508e420bb4a6da9eaeb048fc as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/4807f26a508e420bb4a6da9eaeb048fc 2024-11-11T20:54:49,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/4807f26a508e420bb4a6da9eaeb048fc, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T20:54:49,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/5b53510121a44ce08409a48bdd04ffba as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/5b53510121a44ce08409a48bdd04ffba 2024-11-11T20:54:49,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/5b53510121a44ce08409a48bdd04ffba, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T20:54:49,038 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 82ee51cab8163979e03e6fa77fa08aaf in 547ms, sequenceid=12, compaction requested=false 2024-11-11T20:54:49,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:54:49,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T20:54:49,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:49,093 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 82ee51cab8163979e03e6fa77fa08aaf 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-11T20:54:49,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=A 2024-11-11T20:54:49,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:54:49,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=B 2024-11-11T20:54:49,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:54:49,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=C 2024-11-11T20:54:49,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:54:49,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/0cf4613ac22b4484937b7ed6c5d0e5f5 is 50, key is test_row_0/A:col10/1731358488581/Put/seqid=0 2024-11-11T20:54:49,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742215_1391 (size=12001) 2024-11-11T20:54:49,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T20:54:49,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:54:49,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:49,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:49,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358549237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:49,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358549244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:49,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:49,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358549249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:49,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:49,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358549250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,259 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:49,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:49,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1731358549250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:49,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:49,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358549356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:49,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:49,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358549356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:49,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,366 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:49,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,366 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:49,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:49,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358549365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:49,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:49,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358549365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:49,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:49,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:49,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1731358549365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:49,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:49,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,375 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:49,500 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/0cf4613ac22b4484937b7ed6c5d0e5f5 2024-11-11T20:54:49,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/d14b6faa464b4cf4987067984f6ee38e is 50, key is test_row_0/B:col10/1731358488581/Put/seqid=0 2024-11-11T20:54:49,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742216_1392 (size=12001) 2024-11-11T20:54:49,514 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/d14b6faa464b4cf4987067984f6ee38e 2024-11-11T20:54:49,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/92bf723a400748f9ad42a66bea804870 is 50, key is test_row_0/C:col10/1731358488581/Put/seqid=0 2024-11-11T20:54:49,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742217_1393 (size=12001) 2024-11-11T20:54:49,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358549576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:49,583 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358549576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:49,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358549585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:49,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358549585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:49,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1731358549585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,596 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:49,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,597 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:49,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T20:54:49,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:49,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358549886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:49,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,893 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:49,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:49,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358549895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:49,914 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358549905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:49,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358549905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:49,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1731358549906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:49,915 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:49,925 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/92bf723a400748f9ad42a66bea804870 2024-11-11T20:54:49,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/0cf4613ac22b4484937b7ed6c5d0e5f5 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/0cf4613ac22b4484937b7ed6c5d0e5f5 2024-11-11T20:54:49,932 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/0cf4613ac22b4484937b7ed6c5d0e5f5, entries=150, sequenceid=48, filesize=11.7 K 2024-11-11T20:54:49,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/d14b6faa464b4cf4987067984f6ee38e as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/d14b6faa464b4cf4987067984f6ee38e 2024-11-11T20:54:49,936 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/d14b6faa464b4cf4987067984f6ee38e, entries=150, sequenceid=48, filesize=11.7 K 2024-11-11T20:54:49,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/92bf723a400748f9ad42a66bea804870 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/92bf723a400748f9ad42a66bea804870 2024-11-11T20:54:49,939 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/92bf723a400748f9ad42a66bea804870, entries=150, sequenceid=48, filesize=11.7 K 2024-11-11T20:54:49,940 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=60.38 KB/61830 for 82ee51cab8163979e03e6fa77fa08aaf in 847ms, sequenceid=48, compaction requested=false 2024-11-11T20:54:49,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:54:49,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:49,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-11T20:54:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-11T20:54:49,943 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-11T20:54:49,943 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4590 sec 2024-11-11T20:54:49,944 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.4690 sec 2024-11-11T20:54:50,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:54:50,410 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82ee51cab8163979e03e6fa77fa08aaf 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T20:54:50,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=A 2024-11-11T20:54:50,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:50,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=B 2024-11-11T20:54:50,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:50,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=C 2024-11-11T20:54:50,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:50,414 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/c88566ac1b434f9eb315eb67e68164af is 50, key is test_row_0/A:col10/1731358489248/Put/seqid=0 2024-11-11T20:54:50,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742218_1394 (size=14341) 2024-11-11T20:54:50,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:50,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358550441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:50,452 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:50,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1731358550445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:50,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:50,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358550446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:50,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358550447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:50,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,458 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:50,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358550448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:50,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,459 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:50,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:50,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358550556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:50,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:50,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1731358550556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:50,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,564 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:50,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,564 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:50,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:50,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358550565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:50,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358550566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,574 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:50,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358550566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:50,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:50,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T20:54:50,626 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:54:50,626 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:54:50,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-11T20:54:50,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T20:54:50,627 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:54:50,628 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:54:50,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:54:50,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T20:54:50,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358550766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:50,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:50,780 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T20:54:50,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:50,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:50,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:50,780 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:50,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:50,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:50,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:50,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1731358550776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,783 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:50,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358550786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:50,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358550786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:50,794 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:50,794 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358550786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:50,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:50,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:50,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:50,818 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/c88566ac1b434f9eb315eb67e68164af 2024-11-11T20:54:50,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/0a6c2cdf670b44aeb9a872caa434bf05 is 50, key is test_row_0/B:col10/1731358489248/Put/seqid=0 2024-11-11T20:54:50,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742219_1395 (size=12001) 2024-11-11T20:54:50,932 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T20:54:50,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:50,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:50,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:50,933 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:50,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:50,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:50,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T20:54:51,084 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:51,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358551076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:51,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:51,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:51,084 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:51,085 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T20:54:51,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:51,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:51,085 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:51,086 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:51,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:51,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:51,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:51,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1731358551086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:51,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:51,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:51,088 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:51,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:51,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358551105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:51,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:51,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:51,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358551105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:51,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:51,110 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:51,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:51,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:51,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358551105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:51,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:51,111 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:51,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:51,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:51,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:51,228 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/0a6c2cdf670b44aeb9a872caa434bf05 2024-11-11T20:54:51,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/c782cd80db514fa0b295f89b3137fc2b is 50, key is test_row_0/C:col10/1731358489248/Put/seqid=0 2024-11-11T20:54:51,238 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T20:54:51,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:51,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:51,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:51,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:51,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:51,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742220_1396 (size=12001) 2024-11-11T20:54:51,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:51,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T20:54:51,352 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T20:54:51,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T20:54:51,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:51,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:51,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:51,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:51,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:51,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:51,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T20:54:51,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:51,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:51,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:51,544 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:51,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:51,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:51,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1731358551595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:51,600 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:51,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358551595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:51,600 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:51,601 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:51,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:51,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358551615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:51,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:51,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358551615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:51,622 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:51,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:51,633 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:51,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358551626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:51,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:51,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:51,633 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:51,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/c782cd80db514fa0b295f89b3137fc2b 2024-11-11T20:54:51,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/c88566ac1b434f9eb315eb67e68164af as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/c88566ac1b434f9eb315eb67e68164af 2024-11-11T20:54:51,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/c88566ac1b434f9eb315eb67e68164af, entries=200, sequenceid=61, filesize=14.0 K 2024-11-11T20:54:51,647 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/0a6c2cdf670b44aeb9a872caa434bf05 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/0a6c2cdf670b44aeb9a872caa434bf05 2024-11-11T20:54:51,649 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/0a6c2cdf670b44aeb9a872caa434bf05, entries=150, sequenceid=61, filesize=11.7 K 2024-11-11T20:54:51,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/c782cd80db514fa0b295f89b3137fc2b as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/c782cd80db514fa0b295f89b3137fc2b 2024-11-11T20:54:51,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/c782cd80db514fa0b295f89b3137fc2b, entries=150, sequenceid=61, filesize=11.7 K 2024-11-11T20:54:51,653 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 82ee51cab8163979e03e6fa77fa08aaf in 1243ms, sequenceid=61, compaction requested=true 2024-11-11T20:54:51,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:54:51,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82ee51cab8163979e03e6fa77fa08aaf:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:54:51,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:51,654 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:54:51,654 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:54:51,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82ee51cab8163979e03e6fa77fa08aaf:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:54:51,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:51,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82ee51cab8163979e03e6fa77fa08aaf:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:54:51,654 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:54:51,654 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:54:51,654 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:54:51,654 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 82ee51cab8163979e03e6fa77fa08aaf/B is initiating minor compaction (all files) 2024-11-11T20:54:51,654 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 82ee51cab8163979e03e6fa77fa08aaf/A is initiating minor compaction (all files) 2024-11-11T20:54:51,654 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82ee51cab8163979e03e6fa77fa08aaf/A in TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:51,654 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82ee51cab8163979e03e6fa77fa08aaf/B in TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:51,654 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/3d8380b8e9bd4f7c97b112eb4beef50d, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/0cf4613ac22b4484937b7ed6c5d0e5f5, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/c88566ac1b434f9eb315eb67e68164af] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp, totalSize=37.4 K 2024-11-11T20:54:51,654 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/4807f26a508e420bb4a6da9eaeb048fc, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/d14b6faa464b4cf4987067984f6ee38e, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/0a6c2cdf670b44aeb9a872caa434bf05] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp, totalSize=35.2 K 2024-11-11T20:54:51,655 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 4807f26a508e420bb4a6da9eaeb048fc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731358488487 2024-11-11T20:54:51,655 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d8380b8e9bd4f7c97b112eb4beef50d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731358488487 2024-11-11T20:54:51,655 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting d14b6faa464b4cf4987067984f6ee38e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731358488578 2024-11-11T20:54:51,655 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0cf4613ac22b4484937b7ed6c5d0e5f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731358488578 2024-11-11T20:54:51,655 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a6c2cdf670b44aeb9a872caa434bf05, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731358489248 2024-11-11T20:54:51,655 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting c88566ac1b434f9eb315eb67e68164af, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731358489248 2024-11-11T20:54:51,664 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82ee51cab8163979e03e6fa77fa08aaf#B#compaction#327 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:51,664 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/2fed4524da744f88b5bb44d572a4c93b is 50, key is test_row_0/B:col10/1731358489248/Put/seqid=0 2024-11-11T20:54:51,667 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82ee51cab8163979e03e6fa77fa08aaf#A#compaction#328 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:51,668 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/b3bdf427bccb4409a1d13dbee6c1294f is 50, key is test_row_0/A:col10/1731358489248/Put/seqid=0 2024-11-11T20:54:51,672 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742221_1397 (size=12104) 2024-11-11T20:54:51,679 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/2fed4524da744f88b5bb44d572a4c93b as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/2fed4524da744f88b5bb44d572a4c93b 2024-11-11T20:54:51,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742222_1398 (size=12104) 2024-11-11T20:54:51,685 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82ee51cab8163979e03e6fa77fa08aaf/B of 82ee51cab8163979e03e6fa77fa08aaf into 2fed4524da744f88b5bb44d572a4c93b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:51,685 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:54:51,685 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., storeName=82ee51cab8163979e03e6fa77fa08aaf/B, priority=13, startTime=1731358491654; duration=0sec 2024-11-11T20:54:51,686 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:54:51,686 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82ee51cab8163979e03e6fa77fa08aaf:B 2024-11-11T20:54:51,686 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:54:51,687 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:54:51,687 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 82ee51cab8163979e03e6fa77fa08aaf/C is initiating minor compaction (all files) 2024-11-11T20:54:51,687 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82ee51cab8163979e03e6fa77fa08aaf/C in TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:51,687 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/5b53510121a44ce08409a48bdd04ffba, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/92bf723a400748f9ad42a66bea804870, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/c782cd80db514fa0b295f89b3137fc2b] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp, totalSize=35.2 K 2024-11-11T20:54:51,687 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b53510121a44ce08409a48bdd04ffba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731358488487 2024-11-11T20:54:51,688 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 92bf723a400748f9ad42a66bea804870, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731358488578 2024-11-11T20:54:51,688 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting c782cd80db514fa0b295f89b3137fc2b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731358489248 2024-11-11T20:54:51,693 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/b3bdf427bccb4409a1d13dbee6c1294f as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/b3bdf427bccb4409a1d13dbee6c1294f 2024-11-11T20:54:51,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T20:54:51,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:51,697 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 82ee51cab8163979e03e6fa77fa08aaf 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T20:54:51,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=A 2024-11-11T20:54:51,698 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82ee51cab8163979e03e6fa77fa08aaf/A of 82ee51cab8163979e03e6fa77fa08aaf into b3bdf427bccb4409a1d13dbee6c1294f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:51,698 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:54:51,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:51,698 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., storeName=82ee51cab8163979e03e6fa77fa08aaf/A, priority=13, startTime=1731358491653; duration=0sec 2024-11-11T20:54:51,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=B 2024-11-11T20:54:51,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:51,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=C 2024-11-11T20:54:51,698 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:51,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:51,698 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82ee51cab8163979e03e6fa77fa08aaf:A 2024-11-11T20:54:51,701 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82ee51cab8163979e03e6fa77fa08aaf#C#compaction#329 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:51,702 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/3479b00fad494f57bf5da26f16517faa is 50, key is test_row_0/C:col10/1731358489248/Put/seqid=0 2024-11-11T20:54:51,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/69e6c63fce4245558245ed3b776807f4 is 50, key is test_row_0/A:col10/1731358490446/Put/seqid=0 2024-11-11T20:54:51,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742223_1399 (size=12104) 2024-11-11T20:54:51,722 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/3479b00fad494f57bf5da26f16517faa as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/3479b00fad494f57bf5da26f16517faa 2024-11-11T20:54:51,727 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82ee51cab8163979e03e6fa77fa08aaf/C of 82ee51cab8163979e03e6fa77fa08aaf into 3479b00fad494f57bf5da26f16517faa(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:51,727 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:54:51,727 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., storeName=82ee51cab8163979e03e6fa77fa08aaf/C, priority=13, startTime=1731358491654; duration=0sec 2024-11-11T20:54:51,727 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:51,727 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82ee51cab8163979e03e6fa77fa08aaf:C 2024-11-11T20:54:51,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742224_1400 (size=12001) 2024-11-11T20:54:51,729 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/69e6c63fce4245558245ed3b776807f4 2024-11-11T20:54:51,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/99bd126b8eb8437f8bc59c6c8c29c315 is 50, key is test_row_0/B:col10/1731358490446/Put/seqid=0 2024-11-11T20:54:51,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742225_1401 (size=12001) 2024-11-11T20:54:51,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T20:54:52,138 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/99bd126b8eb8437f8bc59c6c8c29c315 2024-11-11T20:54:52,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/4fe3eb017baa4fe5b867392716b96f24 is 50, key is test_row_0/C:col10/1731358490446/Put/seqid=0 2024-11-11T20:54:52,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742226_1402 (size=12001) 2024-11-11T20:54:52,552 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/4fe3eb017baa4fe5b867392716b96f24 2024-11-11T20:54:52,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/69e6c63fce4245558245ed3b776807f4 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/69e6c63fce4245558245ed3b776807f4 2024-11-11T20:54:52,558 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/69e6c63fce4245558245ed3b776807f4, entries=150, sequenceid=82, filesize=11.7 K 2024-11-11T20:54:52,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/99bd126b8eb8437f8bc59c6c8c29c315 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/99bd126b8eb8437f8bc59c6c8c29c315 2024-11-11T20:54:52,562 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/99bd126b8eb8437f8bc59c6c8c29c315, entries=150, sequenceid=82, filesize=11.7 K 2024-11-11T20:54:52,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/4fe3eb017baa4fe5b867392716b96f24 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/4fe3eb017baa4fe5b867392716b96f24 2024-11-11T20:54:52,565 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/4fe3eb017baa4fe5b867392716b96f24, entries=150, sequenceid=82, filesize=11.7 K 2024-11-11T20:54:52,566 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 82ee51cab8163979e03e6fa77fa08aaf in 869ms, sequenceid=82, compaction requested=false 2024-11-11T20:54:52,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:54:52,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:52,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-11T20:54:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-11T20:54:52,568 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-11T20:54:52,568 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9390 sec 2024-11-11T20:54:52,569 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.9430 sec 2024-11-11T20:54:52,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:54:52,620 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82ee51cab8163979e03e6fa77fa08aaf 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T20:54:52,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=A 2024-11-11T20:54:52,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:52,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=B 2024-11-11T20:54:52,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:52,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=C 2024-11-11T20:54:52,620 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:52,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/715f0e36bbbb4a35846bfc649c6e8950 is 50, key is test_row_0/A:col10/1731358492618/Put/seqid=0 2024-11-11T20:54:52,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742227_1403 (size=14337) 2024-11-11T20:54:52,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358492654, max timestamp=1731358492657Num uniques 859; 2024-11-11T20:54:52,659 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:54:52,660 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358492654, max timestamp=1731358492657Num uniques 859; 2024-11-11T20:54:52,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:54:52,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358492654, max timestamp=1731358492657Num uniques 859; 2024-11-11T20:54:52,661 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:54:52,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358552681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:52,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:52,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358552681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:52,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:52,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:52,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:52,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:52,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358552687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:52,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:52,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:52,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:52,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358552688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:52,693 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:52,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:52,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:52,693 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:52,704 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:52,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1731358552689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:52,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:52,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:52,704 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:52,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T20:54:52,776 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:54:52,776 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:54:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-11T20:54:52,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T20:54:52,778 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:54:52,778 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:54:52,778 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:54:52,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358552796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:52,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:52,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358552796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:52,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:52,804 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:52,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:52,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:52,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:52,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:52,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358552796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:52,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:52,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358552796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:52,805 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:52,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:52,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1731358552815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:52,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:52,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:52,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:52,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T20:54:52,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T20:54:52,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:52,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:52,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:52,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:52,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:53,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:53,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358553005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:53,015 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:53,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:53,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358553015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:53,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:53,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:53,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358553015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:53,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358553015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:53,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/715f0e36bbbb4a35846bfc649c6e8950 2024-11-11T20:54:53,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:53,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1731358553026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:53,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:53,032 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/c50aceec6c6340358aa47d5a8fde36d3 is 50, key is test_row_0/B:col10/1731358492618/Put/seqid=0 2024-11-11T20:54:53,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742228_1404 (size=9657) 2024-11-11T20:54:53,083 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T20:54:53,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:53,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:53,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:53,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:53,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:53,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:53,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T20:54:53,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T20:54:53,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:53,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:53,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:53,236 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:53,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:53,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:53,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:53,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358553325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:53,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,332 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:53,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:53,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358553335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:53,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358553335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:53,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358553336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:53,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1731358553336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,345 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:53,388 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T20:54:53,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:53,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:53,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:53,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:53,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:53,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:53,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T20:54:53,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/c50aceec6c6340358aa47d5a8fde36d3 2024-11-11T20:54:53,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/4845dc028557453b81458a76448b91bd is 50, key is test_row_0/C:col10/1731358492618/Put/seqid=0 2024-11-11T20:54:53,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742229_1405 (size=9657) 2024-11-11T20:54:53,541 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T20:54:53,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:53,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:53,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:53,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:53,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:53,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:53,694 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T20:54:53,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:53,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:53,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:53,694 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:53,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:53,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:53,844 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:53,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358553835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,844 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:53,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T20:54:53,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:53,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:53,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:53,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:53,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:53,847 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/4845dc028557453b81458a76448b91bd 2024-11-11T20:54:53,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:53,850 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/715f0e36bbbb4a35846bfc649c6e8950 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/715f0e36bbbb4a35846bfc649c6e8950 2024-11-11T20:54:53,853 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/715f0e36bbbb4a35846bfc649c6e8950, entries=200, sequenceid=93, filesize=14.0 K 2024-11-11T20:54:53,853 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/c50aceec6c6340358aa47d5a8fde36d3 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/c50aceec6c6340358aa47d5a8fde36d3 2024-11-11T20:54:53,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/c50aceec6c6340358aa47d5a8fde36d3, entries=100, sequenceid=93, filesize=9.4 K 2024-11-11T20:54:53,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/4845dc028557453b81458a76448b91bd as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/4845dc028557453b81458a76448b91bd 2024-11-11T20:54:53,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/4845dc028557453b81458a76448b91bd, entries=100, sequenceid=93, filesize=9.4 K 2024-11-11T20:54:53,860 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 82ee51cab8163979e03e6fa77fa08aaf in 1240ms, sequenceid=93, compaction requested=true 2024-11-11T20:54:53,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:54:53,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82ee51cab8163979e03e6fa77fa08aaf:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:54:53,860 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:54:53,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:53,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82ee51cab8163979e03e6fa77fa08aaf:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:54:53,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:54:53,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82ee51cab8163979e03e6fa77fa08aaf:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:54:53,860 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T20:54:53,861 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38442 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:54:53,861 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 82ee51cab8163979e03e6fa77fa08aaf/A is initiating minor compaction (all files) 2024-11-11T20:54:53,861 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82ee51cab8163979e03e6fa77fa08aaf/A in TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:53,861 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/b3bdf427bccb4409a1d13dbee6c1294f, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/69e6c63fce4245558245ed3b776807f4, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/715f0e36bbbb4a35846bfc649c6e8950] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp, totalSize=37.5 K 2024-11-11T20:54:53,861 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:54:53,862 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting b3bdf427bccb4409a1d13dbee6c1294f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731358489248 2024-11-11T20:54:53,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:54:53,864 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82ee51cab8163979e03e6fa77fa08aaf 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-11T20:54:53,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=A 2024-11-11T20:54:53,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:54:53,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=B 2024-11-11T20:54:53,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:54:53,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=C 2024-11-11T20:54:53,864 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:54:53,865 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69e6c63fce4245558245ed3b776807f4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731358490434 2024-11-11T20:54:53,866 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:54:53,866 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 82ee51cab8163979e03e6fa77fa08aaf/B is initiating minor compaction (all files) 2024-11-11T20:54:53,866 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 715f0e36bbbb4a35846bfc649c6e8950, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731358492612 2024-11-11T20:54:53,866 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82ee51cab8163979e03e6fa77fa08aaf/B in TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:53,866 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/2fed4524da744f88b5bb44d572a4c93b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/99bd126b8eb8437f8bc59c6c8c29c315, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/c50aceec6c6340358aa47d5a8fde36d3] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp, totalSize=33.0 K 2024-11-11T20:54:53,868 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 2fed4524da744f88b5bb44d572a4c93b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731358489248 2024-11-11T20:54:53,869 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 99bd126b8eb8437f8bc59c6c8c29c315, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731358490434 2024-11-11T20:54:53,869 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting c50aceec6c6340358aa47d5a8fde36d3, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731358492618 2024-11-11T20:54:53,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/f7b40cf99573458bb6c8c813e01f0e95 is 50, key is test_row_0/A:col10/1731358493863/Put/seqid=0 2024-11-11T20:54:53,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742230_1406 (size=14391) 2024-11-11T20:54:53,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/f7b40cf99573458bb6c8c813e01f0e95 2024-11-11T20:54:53,881 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82ee51cab8163979e03e6fa77fa08aaf#B#compaction#338 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:53,882 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82ee51cab8163979e03e6fa77fa08aaf#A#compaction#337 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:53,882 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/f2bfd44d96094657922c5103f1f01584 is 50, key is test_row_0/B:col10/1731358492618/Put/seqid=0 2024-11-11T20:54:53,882 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/ea713673f21b415ca536e494f2e6b038 is 50, key is test_row_0/A:col10/1731358492618/Put/seqid=0 2024-11-11T20:54:53,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:53,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358553867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:53,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,886 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:53,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:53,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358553876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:53,887 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:53,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/c4a99ff082524989aeb4d1b72bab85d7 is 50, key is test_row_0/B:col10/1731358493863/Put/seqid=0 2024-11-11T20:54:53,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358553884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:53,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:53,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1731358553886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:53,890 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:53,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:53,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:53,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742232_1408 (size=12207) 2024-11-11T20:54:53,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742233_1409 (size=12051) 2024-11-11T20:54:53,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742231_1407 (size=12207) 2024-11-11T20:54:53,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T20:54:53,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T20:54:53,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:53,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:53,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:53,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:53,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:54,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:54,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358553996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:54,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:54,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358553996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:54,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:54,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:54,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:54,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:54,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1731358553996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:54,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:54,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:54,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:54,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:54,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358553996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:54,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:54,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:54,007 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:54,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T20:54:54,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:54,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:54,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:54,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:54,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:54,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:54,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358554216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:54,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:54,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:54,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358554216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:54,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:54,222 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:54,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:54,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1731358554216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:54,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:54,222 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:54,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:54,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:54,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358554216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:54,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:54,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:54,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:54,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:54,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:54,296 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/ea713673f21b415ca536e494f2e6b038 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/ea713673f21b415ca536e494f2e6b038 2024-11-11T20:54:54,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/c4a99ff082524989aeb4d1b72bab85d7 2024-11-11T20:54:54,300 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82ee51cab8163979e03e6fa77fa08aaf/A of 82ee51cab8163979e03e6fa77fa08aaf into ea713673f21b415ca536e494f2e6b038(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:54,300 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:54:54,300 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., storeName=82ee51cab8163979e03e6fa77fa08aaf/A, priority=13, startTime=1731358493860; duration=0sec 2024-11-11T20:54:54,300 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:54:54,300 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82ee51cab8163979e03e6fa77fa08aaf:A 2024-11-11T20:54:54,300 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:54:54,301 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33762 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:54:54,301 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 82ee51cab8163979e03e6fa77fa08aaf/C is initiating minor compaction (all files) 2024-11-11T20:54:54,301 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82ee51cab8163979e03e6fa77fa08aaf/C in TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:54,301 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/3479b00fad494f57bf5da26f16517faa, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/4fe3eb017baa4fe5b867392716b96f24, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/4845dc028557453b81458a76448b91bd] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp, totalSize=33.0 K 2024-11-11T20:54:54,302 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3479b00fad494f57bf5da26f16517faa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731358489248 2024-11-11T20:54:54,302 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4fe3eb017baa4fe5b867392716b96f24, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731358490434 2024-11-11T20:54:54,302 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4845dc028557453b81458a76448b91bd, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731358492618 2024-11-11T20:54:54,303 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/f2bfd44d96094657922c5103f1f01584 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/f2bfd44d96094657922c5103f1f01584 2024-11-11T20:54:54,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T20:54:54,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:54,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:54,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:54,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:54,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:54,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/5100216d31f345b78e61edafda1e550d is 50, key is test_row_0/C:col10/1731358493863/Put/seqid=0 2024-11-11T20:54:54,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:54,311 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82ee51cab8163979e03e6fa77fa08aaf/B of 82ee51cab8163979e03e6fa77fa08aaf into f2bfd44d96094657922c5103f1f01584(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:54,311 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:54:54,311 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., storeName=82ee51cab8163979e03e6fa77fa08aaf/B, priority=13, startTime=1731358493860; duration=0sec 2024-11-11T20:54:54,311 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:54,311 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82ee51cab8163979e03e6fa77fa08aaf:B 2024-11-11T20:54:54,312 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82ee51cab8163979e03e6fa77fa08aaf#C#compaction#341 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:54,312 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/71c78609c2f2414181c2fd308010838b is 50, key is test_row_0/C:col10/1731358492618/Put/seqid=0 2024-11-11T20:54:54,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742234_1410 (size=12051) 2024-11-11T20:54:54,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742235_1411 (size=12207) 2024-11-11T20:54:54,327 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/71c78609c2f2414181c2fd308010838b as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/71c78609c2f2414181c2fd308010838b 2024-11-11T20:54:54,331 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82ee51cab8163979e03e6fa77fa08aaf/C of 82ee51cab8163979e03e6fa77fa08aaf into 71c78609c2f2414181c2fd308010838b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:54,331 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:54:54,331 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., storeName=82ee51cab8163979e03e6fa77fa08aaf/C, priority=13, startTime=1731358493860; duration=0sec 2024-11-11T20:54:54,331 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:54,331 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82ee51cab8163979e03e6fa77fa08aaf:C 2024-11-11T20:54:54,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T20:54:54,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:54,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:54,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:54,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:54,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:54,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:54,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358554525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:54,531 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:54,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358554525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:54,531 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:54,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:54,532 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:54,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:54,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:54,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358554525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:54,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:54,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:54,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:54,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:54,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:54,543 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:54,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1731358554536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:54,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:54,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:54,543 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:54,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T20:54:54,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:54,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:54,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:54,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:54,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:54,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/5100216d31f345b78e61edafda1e550d 2024-11-11T20:54:54,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/f7b40cf99573458bb6c8c813e01f0e95 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/f7b40cf99573458bb6c8c813e01f0e95 2024-11-11T20:54:54,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/f7b40cf99573458bb6c8c813e01f0e95, entries=200, sequenceid=130, filesize=14.1 K 2024-11-11T20:54:54,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/c4a99ff082524989aeb4d1b72bab85d7 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/c4a99ff082524989aeb4d1b72bab85d7 2024-11-11T20:54:54,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/c4a99ff082524989aeb4d1b72bab85d7, entries=150, sequenceid=130, filesize=11.8 K 2024-11-11T20:54:54,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/5100216d31f345b78e61edafda1e550d as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/5100216d31f345b78e61edafda1e550d 2024-11-11T20:54:54,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/5100216d31f345b78e61edafda1e550d, entries=150, sequenceid=130, filesize=11.8 K 2024-11-11T20:54:54,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=40.25 KB/41220 for 82ee51cab8163979e03e6fa77fa08aaf in 864ms, sequenceid=130, compaction requested=false 2024-11-11T20:54:54,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:54:54,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T20:54:54,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:54,763 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 82ee51cab8163979e03e6fa77fa08aaf 3/3 column families, dataSize=40.25 KB heapSize=120.28 KB 2024-11-11T20:54:54,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=A 2024-11-11T20:54:54,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:54,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=B 2024-11-11T20:54:54,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:54,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=C 2024-11-11T20:54:54,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:54,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/90adf1a23ce048eeb6aa808f3783eee5 is 50, key is test_row_0/A:col10/1731358493883/Put/seqid=0 2024-11-11T20:54:54,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742236_1412 (size=12151) 2024-11-11T20:54:54,771 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/90adf1a23ce048eeb6aa808f3783eee5 2024-11-11T20:54:54,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/eaf83a15e1b4405ca458802b3d953262 is 50, key is test_row_0/B:col10/1731358493883/Put/seqid=0 2024-11-11T20:54:54,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742237_1413 (size=12151) 2024-11-11T20:54:54,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:54:54,856 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T20:54:54,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358494940, max timestamp=1731358494978Num uniques 859; 2024-11-11T20:54:54,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:54:54,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358494940, max timestamp=1731358494978Num uniques 859; 2024-11-11T20:54:54,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:54:54,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358494940, max timestamp=1731358494978Num uniques 859; 2024-11-11T20:54:54,981 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:54:55,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:55,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358555048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:55,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,062 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:55,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1731358555055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:55,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358555056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:55,064 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:55,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,065 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:55,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:55,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358555057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:55,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,069 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:55,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,069 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:55,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358555062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:55,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,069 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:55,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:55,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358555165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:55,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,174 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:55,181 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/eaf83a15e1b4405ca458802b3d953262 2024-11-11T20:54:55,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/15f1dc900dee46b583a99de2997d27a9 is 50, key is test_row_0/C:col10/1731358493883/Put/seqid=0 2024-11-11T20:54:55,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:55,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358555175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:55,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:55,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358555176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:55,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358555176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,188 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:55,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742238_1414 (size=12151) 2024-11-11T20:54:55,394 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:55,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358555385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:55,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,395 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:55,404 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:55,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358555395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:55,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:55,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358555395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:55,405 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:55,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,405 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:55,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,405 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:55,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358555396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:55,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:55,589 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/15f1dc900dee46b583a99de2997d27a9 2024-11-11T20:54:55,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/90adf1a23ce048eeb6aa808f3783eee5 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/90adf1a23ce048eeb6aa808f3783eee5 2024-11-11T20:54:55,595 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/90adf1a23ce048eeb6aa808f3783eee5, entries=150, sequenceid=142, filesize=11.9 K 2024-11-11T20:54:55,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/eaf83a15e1b4405ca458802b3d953262 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/eaf83a15e1b4405ca458802b3d953262 2024-11-11T20:54:55,599 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/eaf83a15e1b4405ca458802b3d953262, entries=150, sequenceid=142, filesize=11.9 K 2024-11-11T20:54:55,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/15f1dc900dee46b583a99de2997d27a9 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/15f1dc900dee46b583a99de2997d27a9 2024-11-11T20:54:55,603 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/15f1dc900dee46b583a99de2997d27a9, entries=150, sequenceid=142, filesize=11.9 K 2024-11-11T20:54:55,604 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~120.23 KB/123120, currentSize=140.89 KB/144270 for 82ee51cab8163979e03e6fa77fa08aaf in 841ms, sequenceid=142, compaction requested=true 2024-11-11T20:54:55,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:54:55,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:55,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-11T20:54:55,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-11T20:54:55,607 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-11T20:54:55,607 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8270 sec 2024-11-11T20:54:55,608 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 2.8310 sec 2024-11-11T20:54:55,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:54:55,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82ee51cab8163979e03e6fa77fa08aaf 3/3 column families, dataSize=147.60 KB heapSize=420.84 KB 2024-11-11T20:54:55,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=A 2024-11-11T20:54:55,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T20:54:55,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=B 2024-11-11T20:54:55,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T20:54:55,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=C 2024-11-11T20:54:55,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T20:54:55,715 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/cfcb1f8b4c444fa1a797cdaef792b2bd is 50, key is test_row_0/A:col10/1731358495060/Put/seqid=0 2024-11-11T20:54:55,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742239_1415 (size=14541) 2024-11-11T20:54:55,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:55,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358555723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:55,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358555724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:55,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358555725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,730 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:55,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:55,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358555727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:55,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,742 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:55,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:55,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358555835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:55,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:55,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,844 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358555835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:55,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:55,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,844 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:55,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358555836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:55,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,844 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:55,854 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:55,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358555846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:55,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:55,855 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:56,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:56,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358556045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:56,055 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:56,065 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358556056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:56,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:56,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:56,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:56,066 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358556056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:56,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:56,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:56,066 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:56,067 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358556066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:56,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:56,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:56,068 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:56,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:56,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1731358556076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:56,079 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_0' in 82ee51cab8163979e03e6fa77fa08aaf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:54:56,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:56,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:56,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:56,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/cfcb1f8b4c444fa1a797cdaef792b2bd 2024-11-11T20:54:56,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/720fbaaebb52455fa26464a4572feae3 is 50, key is test_row_0/B:col10/1731358495060/Put/seqid=0 2024-11-11T20:54:56,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742240_1416 (size=12151) 2024-11-11T20:54:56,373 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:56,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358556365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:56,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:56,373 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:56,374 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:56,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:56,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358556375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:56,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:56,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:56,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:56,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358556375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:56,388 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:56,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:56,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:56,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:56,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358556375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:56,388 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:56,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:56,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:56,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:56,528 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/720fbaaebb52455fa26464a4572feae3 2024-11-11T20:54:56,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/93f4dfcaa53e4a0286e8299ec947ca3d is 50, key is test_row_0/C:col10/1731358495060/Put/seqid=0 2024-11-11T20:54:56,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742241_1417 (size=12151) 2024-11-11T20:54:56,884 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:56,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358556875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:56,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:56,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:56,885 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:56,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:56,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358556896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:56,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:56,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:56,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:56,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358556896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:56,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:56,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:56,903 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:56,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:56,903 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:56,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358556896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:56,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:56,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:56,903 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:56,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/93f4dfcaa53e4a0286e8299ec947ca3d 2024-11-11T20:54:56,941 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/cfcb1f8b4c444fa1a797cdaef792b2bd as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/cfcb1f8b4c444fa1a797cdaef792b2bd 2024-11-11T20:54:56,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/cfcb1f8b4c444fa1a797cdaef792b2bd, entries=200, sequenceid=180, filesize=14.2 K 2024-11-11T20:54:56,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/720fbaaebb52455fa26464a4572feae3 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/720fbaaebb52455fa26464a4572feae3 2024-11-11T20:54:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T20:54:56,945 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:54:56,946 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:54:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-11T20:54:56,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T20:54:56,948 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:54:56,948 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:54:56,948 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:54:56,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/720fbaaebb52455fa26464a4572feae3, entries=150, sequenceid=180, filesize=11.9 K 2024-11-11T20:54:56,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/93f4dfcaa53e4a0286e8299ec947ca3d as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/93f4dfcaa53e4a0286e8299ec947ca3d 2024-11-11T20:54:56,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/93f4dfcaa53e4a0286e8299ec947ca3d, entries=150, sequenceid=180, filesize=11.9 K 2024-11-11T20:54:56,955 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~420.80 KB/430896, currentSize=33.54 KB/34350 for 82ee51cab8163979e03e6fa77fa08aaf in 1245ms, sequenceid=180, compaction requested=true 2024-11-11T20:54:56,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:54:56,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82ee51cab8163979e03e6fa77fa08aaf:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:54:56,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:56,955 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:54:56,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82ee51cab8163979e03e6fa77fa08aaf:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:54:56,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:54:56,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82ee51cab8163979e03e6fa77fa08aaf:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:54:56,955 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:54:56,955 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:54:56,956 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53290 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:54:56,956 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 82ee51cab8163979e03e6fa77fa08aaf/A is initiating minor compaction (all files) 2024-11-11T20:54:56,956 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48560 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:54:56,957 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 82ee51cab8163979e03e6fa77fa08aaf/B is initiating minor compaction (all files) 2024-11-11T20:54:56,957 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82ee51cab8163979e03e6fa77fa08aaf/A in TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:56,957 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82ee51cab8163979e03e6fa77fa08aaf/B in TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:56,957 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/ea713673f21b415ca536e494f2e6b038, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/f7b40cf99573458bb6c8c813e01f0e95, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/90adf1a23ce048eeb6aa808f3783eee5, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/cfcb1f8b4c444fa1a797cdaef792b2bd] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp, totalSize=52.0 K 2024-11-11T20:54:56,957 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/f2bfd44d96094657922c5103f1f01584, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/c4a99ff082524989aeb4d1b72bab85d7, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/eaf83a15e1b4405ca458802b3d953262, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/720fbaaebb52455fa26464a4572feae3] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp, totalSize=47.4 K 2024-11-11T20:54:56,957 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea713673f21b415ca536e494f2e6b038, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731358490434 2024-11-11T20:54:56,957 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting f2bfd44d96094657922c5103f1f01584, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731358490434 2024-11-11T20:54:56,957 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting c4a99ff082524989aeb4d1b72bab85d7, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731358492685 2024-11-11T20:54:56,957 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting f7b40cf99573458bb6c8c813e01f0e95, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731358492680 2024-11-11T20:54:56,958 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting eaf83a15e1b4405ca458802b3d953262, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731358493874 2024-11-11T20:54:56,958 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90adf1a23ce048eeb6aa808f3783eee5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731358493874 2024-11-11T20:54:56,958 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 720fbaaebb52455fa26464a4572feae3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731358495054 2024-11-11T20:54:56,958 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfcb1f8b4c444fa1a797cdaef792b2bd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731358495046 2024-11-11T20:54:56,966 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82ee51cab8163979e03e6fa77fa08aaf#B#compaction#348 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:56,966 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82ee51cab8163979e03e6fa77fa08aaf#A#compaction#349 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:56,966 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/85a4dc4b7a9748cc9b078cc180f4a859 is 50, key is test_row_0/B:col10/1731358495060/Put/seqid=0 2024-11-11T20:54:56,967 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/2eabdd223d734730b7bf60f177779125 is 50, key is test_row_0/A:col10/1731358495060/Put/seqid=0 2024-11-11T20:54:56,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742242_1418 (size=12493) 2024-11-11T20:54:56,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742243_1419 (size=12493) 2024-11-11T20:54:56,983 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/85a4dc4b7a9748cc9b078cc180f4a859 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/85a4dc4b7a9748cc9b078cc180f4a859 2024-11-11T20:54:56,983 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/2eabdd223d734730b7bf60f177779125 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/2eabdd223d734730b7bf60f177779125 2024-11-11T20:54:56,987 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 82ee51cab8163979e03e6fa77fa08aaf/A of 82ee51cab8163979e03e6fa77fa08aaf into 2eabdd223d734730b7bf60f177779125(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:56,987 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 82ee51cab8163979e03e6fa77fa08aaf/B of 82ee51cab8163979e03e6fa77fa08aaf into 85a4dc4b7a9748cc9b078cc180f4a859(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:56,987 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:54:56,987 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:54:56,987 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., storeName=82ee51cab8163979e03e6fa77fa08aaf/A, priority=12, startTime=1731358496955; duration=0sec 2024-11-11T20:54:56,987 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., storeName=82ee51cab8163979e03e6fa77fa08aaf/B, priority=12, startTime=1731358496955; duration=0sec 2024-11-11T20:54:56,987 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:54:56,987 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:54:56,987 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82ee51cab8163979e03e6fa77fa08aaf:A 2024-11-11T20:54:56,987 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82ee51cab8163979e03e6fa77fa08aaf:B 2024-11-11T20:54:56,987 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:54:56,988 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48560 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:54:56,988 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 82ee51cab8163979e03e6fa77fa08aaf/C is initiating minor compaction (all files) 2024-11-11T20:54:56,988 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82ee51cab8163979e03e6fa77fa08aaf/C in TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:56,988 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/71c78609c2f2414181c2fd308010838b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/5100216d31f345b78e61edafda1e550d, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/15f1dc900dee46b583a99de2997d27a9, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/93f4dfcaa53e4a0286e8299ec947ca3d] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp, totalSize=47.4 K 2024-11-11T20:54:56,989 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71c78609c2f2414181c2fd308010838b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731358490434 2024-11-11T20:54:56,989 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5100216d31f345b78e61edafda1e550d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1731358492685 2024-11-11T20:54:56,989 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 15f1dc900dee46b583a99de2997d27a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731358493874 2024-11-11T20:54:56,990 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93f4dfcaa53e4a0286e8299ec947ca3d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731358495054 2024-11-11T20:54:56,996 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82ee51cab8163979e03e6fa77fa08aaf#C#compaction#350 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:56,997 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/529822b96634434aa9778ab3c44e9e66 is 50, key is test_row_0/C:col10/1731358495060/Put/seqid=0 2024-11-11T20:54:57,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742244_1420 (size=12493) 2024-11-11T20:54:57,018 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/529822b96634434aa9778ab3c44e9e66 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/529822b96634434aa9778ab3c44e9e66 2024-11-11T20:54:57,024 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 82ee51cab8163979e03e6fa77fa08aaf/C of 82ee51cab8163979e03e6fa77fa08aaf into 529822b96634434aa9778ab3c44e9e66(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:54:57,024 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:54:57,024 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., storeName=82ee51cab8163979e03e6fa77fa08aaf/C, priority=12, startTime=1731358496955; duration=0sec 2024-11-11T20:54:57,024 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:57,024 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82ee51cab8163979e03e6fa77fa08aaf:C 2024-11-11T20:54:57,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T20:54:57,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T20:54:57,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:57,101 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 82ee51cab8163979e03e6fa77fa08aaf 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T20:54:57,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=A 2024-11-11T20:54:57,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:57,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=B 2024-11-11T20:54:57,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:57,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=C 2024-11-11T20:54:57,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:54:57,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/0c2bdd244a7f45beb59ac6f53ce65921 is 50, key is test_row_0/A:col10/1731358495726/Put/seqid=0 2024-11-11T20:54:57,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742245_1421 (size=12151) 2024-11-11T20:54:57,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T20:54:57,509 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/0c2bdd244a7f45beb59ac6f53ce65921 2024-11-11T20:54:57,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/a05f338268fb44dabe5c1ad344b0fbc5 is 50, key is test_row_0/B:col10/1731358495726/Put/seqid=0 2024-11-11T20:54:57,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742246_1422 (size=12151) 2024-11-11T20:54:57,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T20:54:57,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:54:57,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:57,920 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/a05f338268fb44dabe5c1ad344b0fbc5 2024-11-11T20:54:57,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/b1c7b6a430f840719d3a08793a2bfda2 is 50, key is test_row_0/C:col10/1731358495726/Put/seqid=0 2024-11-11T20:54:57,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742247_1423 (size=12151) 2024-11-11T20:54:57,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731358497923, max timestamp=1731358497941Num uniques 859; 2024-11-11T20:54:57,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731358497923, max timestamp=1731358497941Num uniques 859; 2024-11-11T20:54:57,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731358497923, max timestamp=1731358497941Num uniques 859; 2024-11-11T20:54:57,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T20:54:57,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T20:54:57,944 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T20:54:57,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:57,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358557974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:57,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:57,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:57,985 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:57,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:57,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358557982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:57,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:57,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:57,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:57,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:57,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358557983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:57,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:57,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:57,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:57,995 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:57,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358557984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:57,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:57,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:57,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:58,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T20:54:58,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:58,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1731358558095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:58,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:58,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358558096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:58,102 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358558096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:58,101 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_0' in 82ee51cab8163979e03e6fa77fa08aaf of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:54:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:58,102 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:58,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:58,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358558106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:58,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:58,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358558106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:58,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:58,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:58,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:58,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358558306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:58,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:58,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358558306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:58,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,314 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:58,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:58,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:58,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358558316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:58,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358558316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:58,330 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/b1c7b6a430f840719d3a08793a2bfda2 2024-11-11T20:54:58,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/0c2bdd244a7f45beb59ac6f53ce65921 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/0c2bdd244a7f45beb59ac6f53ce65921 2024-11-11T20:54:58,336 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/0c2bdd244a7f45beb59ac6f53ce65921, entries=150, sequenceid=191, filesize=11.9 K 2024-11-11T20:54:58,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/a05f338268fb44dabe5c1ad344b0fbc5 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/a05f338268fb44dabe5c1ad344b0fbc5 2024-11-11T20:54:58,340 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/a05f338268fb44dabe5c1ad344b0fbc5, entries=150, sequenceid=191, filesize=11.9 K 2024-11-11T20:54:58,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/b1c7b6a430f840719d3a08793a2bfda2 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/b1c7b6a430f840719d3a08793a2bfda2 2024-11-11T20:54:58,343 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/b1c7b6a430f840719d3a08793a2bfda2, entries=150, sequenceid=191, filesize=11.9 K 2024-11-11T20:54:58,344 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=154.31 KB/158010 for 82ee51cab8163979e03e6fa77fa08aaf in 1243ms, sequenceid=191, compaction requested=false 2024-11-11T20:54:58,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:54:58,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:58,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-11T20:54:58,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-11T20:54:58,347 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-11T20:54:58,347 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3980 sec 2024-11-11T20:54:58,349 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.4020 sec 2024-11-11T20:54:58,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:54:58,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82ee51cab8163979e03e6fa77fa08aaf 3/3 column families, dataSize=161.02 KB heapSize=446.16 KB 2024-11-11T20:54:58,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=A 2024-11-11T20:54:58,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T20:54:58,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=B 2024-11-11T20:54:58,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T20:54:58,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=C 2024-11-11T20:54:58,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T20:54:58,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/291aae84d7d14a5da0bec946598c9b81 is 50, key is test_row_0/A:col10/1731358497981/Put/seqid=0 2024-11-11T20:54:58,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742248_1424 (size=14541) 2024-11-11T20:54:58,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:58,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358558646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:58,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:58,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358558647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:58,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,649 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:58,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:58,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:58,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358558647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:58,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:58,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358558648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:58,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:58,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:58,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358558755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:58,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358558755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:58,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:58,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358558765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:58,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:58,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358558765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:58,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:58,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,772 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:58,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:58,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358558966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:58,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:58,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358558966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:58,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:58,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,972 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:58,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:58,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358558975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:58,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358558975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:58,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:59,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/291aae84d7d14a5da0bec946598c9b81 2024-11-11T20:54:59,048 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/3b755b6c11664405ab40ba57737e5680 is 50, key is test_row_0/B:col10/1731358497981/Put/seqid=0 2024-11-11T20:54:59,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742249_1425 (size=12151) 2024-11-11T20:54:59,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T20:54:59,106 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:54:59,106 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:54:59,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-11T20:54:59,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T20:54:59,108 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:54:59,108 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:54:59,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:54:59,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T20:54:59,260 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T20:54:59,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:59,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:59,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:59,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:59,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:59,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:59,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:59,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358559275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:59,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:59,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:59,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358559275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:59,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:59,281 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:59,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:59,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:59,281 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:59,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358559285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:59,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:59,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358559285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:59,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:59,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:59,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:59,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:59,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:59,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:59,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T20:54:59,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:59,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:59,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:59,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:59,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:59,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T20:54:59,454 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/3b755b6c11664405ab40ba57737e5680 2024-11-11T20:54:59,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/0ebb40669bec4c9fb1679d6af85ee004 is 50, key is test_row_0/C:col10/1731358497981/Put/seqid=0 2024-11-11T20:54:59,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742250_1426 (size=12151) 2024-11-11T20:54:59,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T20:54:59,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:59,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:59,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:59,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:59,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:59,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:59,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T20:54:59,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:59,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:59,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:59,718 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:59,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:59,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:59,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T20:54:59,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:59,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358559785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:59,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:59,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:59,792 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:59,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:59,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358559795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:59,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:59,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:59,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:59,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:59,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358559796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:59,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:54:59,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:59,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:59,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:59,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358559796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:54:59,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:59,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:54:59,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:54:59,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/0ebb40669bec4c9fb1679d6af85ee004 2024-11-11T20:54:59,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T20:54:59,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:59,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:54:59,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:59,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:59,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:59,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:54:59,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/291aae84d7d14a5da0bec946598c9b81 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/291aae84d7d14a5da0bec946598c9b81 2024-11-11T20:54:59,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/291aae84d7d14a5da0bec946598c9b81, entries=200, sequenceid=229, filesize=14.2 K 2024-11-11T20:54:59,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/3b755b6c11664405ab40ba57737e5680 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/3b755b6c11664405ab40ba57737e5680 2024-11-11T20:54:59,881 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/3b755b6c11664405ab40ba57737e5680, entries=150, sequenceid=229, filesize=11.9 K 2024-11-11T20:54:59,882 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/0ebb40669bec4c9fb1679d6af85ee004 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/0ebb40669bec4c9fb1679d6af85ee004 2024-11-11T20:54:59,885 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/0ebb40669bec4c9fb1679d6af85ee004, entries=150, sequenceid=229, filesize=11.9 K 2024-11-11T20:54:59,886 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~466.50 KB/477696, currentSize=26.84 KB/27480 for 82ee51cab8163979e03e6fa77fa08aaf in 1253ms, sequenceid=229, compaction requested=true 2024-11-11T20:54:59,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:54:59,886 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82ee51cab8163979e03e6fa77fa08aaf:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:54:59,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:59,887 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:54:59,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82ee51cab8163979e03e6fa77fa08aaf:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:54:59,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:54:59,887 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:54:59,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82ee51cab8163979e03e6fa77fa08aaf:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:54:59,887 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:54:59,887 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:54:59,887 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:54:59,887 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 82ee51cab8163979e03e6fa77fa08aaf/B is initiating minor compaction (all files) 2024-11-11T20:54:59,887 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 82ee51cab8163979e03e6fa77fa08aaf/A is initiating minor compaction (all files) 2024-11-11T20:54:59,887 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82ee51cab8163979e03e6fa77fa08aaf/B in TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:59,887 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82ee51cab8163979e03e6fa77fa08aaf/A in TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:54:59,887 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/2eabdd223d734730b7bf60f177779125, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/0c2bdd244a7f45beb59ac6f53ce65921, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/291aae84d7d14a5da0bec946598c9b81] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp, totalSize=38.3 K 2024-11-11T20:54:59,888 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/85a4dc4b7a9748cc9b078cc180f4a859, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/a05f338268fb44dabe5c1ad344b0fbc5, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/3b755b6c11664405ab40ba57737e5680] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp, totalSize=35.9 K 2024-11-11T20:54:59,888 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 85a4dc4b7a9748cc9b078cc180f4a859, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731358495054 2024-11-11T20:54:59,888 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2eabdd223d734730b7bf60f177779125, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731358495054 2024-11-11T20:54:59,889 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting a05f338268fb44dabe5c1ad344b0fbc5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731358495722 2024-11-11T20:54:59,889 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c2bdd244a7f45beb59ac6f53ce65921, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731358495722 2024-11-11T20:54:59,889 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b755b6c11664405ab40ba57737e5680, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731358497981 2024-11-11T20:54:59,890 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 291aae84d7d14a5da0bec946598c9b81, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731358497981 2024-11-11T20:54:59,897 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82ee51cab8163979e03e6fa77fa08aaf#B#compaction#357 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:59,897 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/baaf610561ad4b03afa4ccd7b8a85b85 is 50, key is test_row_0/B:col10/1731358497981/Put/seqid=0 2024-11-11T20:54:59,900 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82ee51cab8163979e03e6fa77fa08aaf#A#compaction#358 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:54:59,901 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/ae323a0ddca347a0a67aa40f63357913 is 50, key is test_row_0/A:col10/1731358497981/Put/seqid=0 2024-11-11T20:54:59,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742252_1428 (size=12595) 2024-11-11T20:54:59,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742251_1427 (size=12595) 2024-11-11T20:55:00,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T20:55:00,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:00,024 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 82ee51cab8163979e03e6fa77fa08aaf 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-11T20:55:00,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=A 2024-11-11T20:55:00,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:00,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=B 2024-11-11T20:55:00,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:00,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=C 2024-11-11T20:55:00,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:00,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/b30ae60e532042e29e73636dd64fe114 is 50, key is test_row_0/A:col10/1731358498647/Put/seqid=0 2024-11-11T20:55:00,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742253_1429 (size=9757) 2024-11-11T20:55:00,033 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/b30ae60e532042e29e73636dd64fe114 2024-11-11T20:55:00,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/51a0f4264c224129a665a2c935659fa7 is 50, key is test_row_0/B:col10/1731358498647/Put/seqid=0 2024-11-11T20:55:00,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742254_1430 (size=9757) 2024-11-11T20:55:00,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T20:55:00,319 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/ae323a0ddca347a0a67aa40f63357913 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/ae323a0ddca347a0a67aa40f63357913 2024-11-11T20:55:00,319 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/baaf610561ad4b03afa4ccd7b8a85b85 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/baaf610561ad4b03afa4ccd7b8a85b85 2024-11-11T20:55:00,324 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82ee51cab8163979e03e6fa77fa08aaf/A of 82ee51cab8163979e03e6fa77fa08aaf into ae323a0ddca347a0a67aa40f63357913(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:55:00,324 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:55:00,324 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., storeName=82ee51cab8163979e03e6fa77fa08aaf/A, priority=13, startTime=1731358499886; duration=0sec 2024-11-11T20:55:00,324 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:55:00,324 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82ee51cab8163979e03e6fa77fa08aaf:A 2024-11-11T20:55:00,324 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:55:00,325 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82ee51cab8163979e03e6fa77fa08aaf/B of 82ee51cab8163979e03e6fa77fa08aaf into baaf610561ad4b03afa4ccd7b8a85b85(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:55:00,325 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:55:00,325 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., storeName=82ee51cab8163979e03e6fa77fa08aaf/B, priority=13, startTime=1731358499887; duration=0sec 2024-11-11T20:55:00,325 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:00,325 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82ee51cab8163979e03e6fa77fa08aaf:B 2024-11-11T20:55:00,325 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:55:00,325 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 82ee51cab8163979e03e6fa77fa08aaf/C is initiating minor compaction (all files) 2024-11-11T20:55:00,325 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82ee51cab8163979e03e6fa77fa08aaf/C in TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:00,325 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/529822b96634434aa9778ab3c44e9e66, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/b1c7b6a430f840719d3a08793a2bfda2, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/0ebb40669bec4c9fb1679d6af85ee004] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp, totalSize=35.9 K 2024-11-11T20:55:00,326 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 529822b96634434aa9778ab3c44e9e66, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731358495054 2024-11-11T20:55:00,326 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1c7b6a430f840719d3a08793a2bfda2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731358495722 2024-11-11T20:55:00,326 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ebb40669bec4c9fb1679d6af85ee004, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731358497981 2024-11-11T20:55:00,332 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82ee51cab8163979e03e6fa77fa08aaf#C#compaction#361 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:55:00,332 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/2d6b8267c03c41609b42e5c90f428fd5 is 50, key is test_row_0/C:col10/1731358497981/Put/seqid=0 2024-11-11T20:55:00,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742255_1431 (size=12595) 2024-11-11T20:55:00,343 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/2d6b8267c03c41609b42e5c90f428fd5 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/2d6b8267c03c41609b42e5c90f428fd5 2024-11-11T20:55:00,350 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82ee51cab8163979e03e6fa77fa08aaf/C of 82ee51cab8163979e03e6fa77fa08aaf into 2d6b8267c03c41609b42e5c90f428fd5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:55:00,351 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:55:00,351 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., storeName=82ee51cab8163979e03e6fa77fa08aaf/C, priority=13, startTime=1731358499887; duration=0sec 2024-11-11T20:55:00,351 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:00,351 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82ee51cab8163979e03e6fa77fa08aaf:C 2024-11-11T20:55:00,452 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/51a0f4264c224129a665a2c935659fa7 2024-11-11T20:55:00,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/8f501d2ab60947eca1e464fe50584f91 is 50, key is test_row_0/C:col10/1731358498647/Put/seqid=0 2024-11-11T20:55:00,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742256_1432 (size=9757) 2024-11-11T20:55:00,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:55:00,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:55:00,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358500829, max timestamp=1731358500838Num uniques 859; 2024-11-11T20:55:00,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:55:00,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358500829, max timestamp=1731358500838Num uniques 859; 2024-11-11T20:55:00,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:55:00,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358500829, max timestamp=1731358500838Num uniques 859; 2024-11-11T20:55:00,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:55:00,862 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/8f501d2ab60947eca1e464fe50584f91 2024-11-11T20:55:00,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/b30ae60e532042e29e73636dd64fe114 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/b30ae60e532042e29e73636dd64fe114 2024-11-11T20:55:00,868 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/b30ae60e532042e29e73636dd64fe114, entries=100, sequenceid=236, filesize=9.5 K 2024-11-11T20:55:00,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/51a0f4264c224129a665a2c935659fa7 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/51a0f4264c224129a665a2c935659fa7 2024-11-11T20:55:00,871 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/51a0f4264c224129a665a2c935659fa7, entries=100, sequenceid=236, filesize=9.5 K 2024-11-11T20:55:00,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/8f501d2ab60947eca1e464fe50584f91 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/8f501d2ab60947eca1e464fe50584f91 2024-11-11T20:55:00,875 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/8f501d2ab60947eca1e464fe50584f91, entries=100, sequenceid=236, filesize=9.5 K 2024-11-11T20:55:00,875 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=120.76 KB/123660 for 82ee51cab8163979e03e6fa77fa08aaf in 851ms, sequenceid=236, compaction requested=false 2024-11-11T20:55:00,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:55:00,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:00,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:55:00,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-11T20:55:00,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82ee51cab8163979e03e6fa77fa08aaf 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-11T20:55:00,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=A 2024-11-11T20:55:00,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:55:00,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=B 2024-11-11T20:55:00,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:55:00,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=C 2024-11-11T20:55:00,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:55:00,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-11T20:55:00,879 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-11T20:55:00,879 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7690 sec 2024-11-11T20:55:00,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/9438f70ecc654900b137269714159dcc is 50, key is test_row_0/A:col10/1731358500875/Put/seqid=0 2024-11-11T20:55:00,881 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 1.7730 sec 2024-11-11T20:55:00,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742257_1433 (size=19621) 2024-11-11T20:55:00,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:00,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358560890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:00,898 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:00,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358560892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:00,898 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:00,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:00,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358560893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:00,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:00,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:00,904 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:00,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:00,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358560898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:00,906 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:01,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:01,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358561005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:01,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:01,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:01,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:01,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358561005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:01,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:01,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:01,012 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:01,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358561005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:01,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:01,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:01,012 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:01,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:01,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358561016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:01,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:01,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:01,025 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:01,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:01,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358561215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:01,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:01,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:01,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:01,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358561216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:01,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:01,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:01,228 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:01,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:01,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358561216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:01,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:01,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:01,228 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:01,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:01,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358561236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:01,244 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T20:55:01,256 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:55:01,256 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:55:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-11T20:55:01,257 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:55:01,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T20:55:01,258 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:55:01,258 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:55:01,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/9438f70ecc654900b137269714159dcc 2024-11-11T20:55:01,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/e00e25358a8f4a1f931c231477f21b5c is 50, key is test_row_0/B:col10/1731358500875/Put/seqid=0 2024-11-11T20:55:01,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742258_1434 (size=12301) 2024-11-11T20:55:01,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T20:55:01,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T20:55:01,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:01,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:55:01,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:01,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:01,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:01,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:01,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:01,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358561535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:01,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:01,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:01,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:01,540 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358561536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:01,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:01,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:01,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:01,540 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:01,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358561536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:01,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:01,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:01,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:01,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T20:55:01,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:01,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:55:01,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:01,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:01,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:01,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358561556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:01,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:01,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:01,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:01,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:01,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T20:55:01,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/e00e25358a8f4a1f931c231477f21b5c 2024-11-11T20:55:01,707 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/35902a21d15b49bba3e242b4790657ed is 50, key is test_row_0/C:col10/1731358500875/Put/seqid=0 2024-11-11T20:55:01,710 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742259_1435 (size=12301) 2024-11-11T20:55:01,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/35902a21d15b49bba3e242b4790657ed 2024-11-11T20:55:01,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T20:55:01,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:01,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:55:01,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:01,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:01,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:01,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:01,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/9438f70ecc654900b137269714159dcc as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/9438f70ecc654900b137269714159dcc 2024-11-11T20:55:01,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/9438f70ecc654900b137269714159dcc, entries=300, sequenceid=274, filesize=19.2 K 2024-11-11T20:55:01,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/e00e25358a8f4a1f931c231477f21b5c as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/e00e25358a8f4a1f931c231477f21b5c 2024-11-11T20:55:01,726 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/e00e25358a8f4a1f931c231477f21b5c, entries=150, sequenceid=274, filesize=12.0 K 2024-11-11T20:55:01,726 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/35902a21d15b49bba3e242b4790657ed as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/35902a21d15b49bba3e242b4790657ed 2024-11-11T20:55:01,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/35902a21d15b49bba3e242b4790657ed, entries=150, sequenceid=274, filesize=12.0 K 2024-11-11T20:55:01,731 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 82ee51cab8163979e03e6fa77fa08aaf in 856ms, sequenceid=274, compaction requested=true 2024-11-11T20:55:01,731 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:55:01,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82ee51cab8163979e03e6fa77fa08aaf:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:55:01,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:01,731 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:55:01,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82ee51cab8163979e03e6fa77fa08aaf:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:55:01,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:01,731 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:55:01,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82ee51cab8163979e03e6fa77fa08aaf:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:55:01,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:55:01,732 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41973 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:55:01,732 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:55:01,732 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 82ee51cab8163979e03e6fa77fa08aaf/A is initiating minor compaction (all files) 2024-11-11T20:55:01,732 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 82ee51cab8163979e03e6fa77fa08aaf/B is initiating minor compaction (all files) 2024-11-11T20:55:01,732 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82ee51cab8163979e03e6fa77fa08aaf/A in TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:01,732 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82ee51cab8163979e03e6fa77fa08aaf/B in TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:01,732 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/ae323a0ddca347a0a67aa40f63357913, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/b30ae60e532042e29e73636dd64fe114, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/9438f70ecc654900b137269714159dcc] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp, totalSize=41.0 K 2024-11-11T20:55:01,732 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/baaf610561ad4b03afa4ccd7b8a85b85, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/51a0f4264c224129a665a2c935659fa7, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/e00e25358a8f4a1f931c231477f21b5c] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp, totalSize=33.8 K 2024-11-11T20:55:01,733 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting baaf610561ad4b03afa4ccd7b8a85b85, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731358497981 2024-11-11T20:55:01,733 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae323a0ddca347a0a67aa40f63357913, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731358497981 2024-11-11T20:55:01,733 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 51a0f4264c224129a665a2c935659fa7, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731358498646 2024-11-11T20:55:01,733 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting b30ae60e532042e29e73636dd64fe114, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731358498646 2024-11-11T20:55:01,733 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting e00e25358a8f4a1f931c231477f21b5c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731358500854 2024-11-11T20:55:01,733 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9438f70ecc654900b137269714159dcc, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731358500854 2024-11-11T20:55:01,741 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82ee51cab8163979e03e6fa77fa08aaf#B#compaction#367 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:55:01,741 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82ee51cab8163979e03e6fa77fa08aaf#A#compaction#366 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:55:01,741 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/1854c8b3da85402eb285617e675249b9 is 50, key is test_row_0/B:col10/1731358500875/Put/seqid=0 2024-11-11T20:55:01,741 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/8ff6d89c71ec42d4b1ac46f9764e1545 is 50, key is test_row_0/A:col10/1731358500875/Put/seqid=0 2024-11-11T20:55:01,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742260_1436 (size=12847) 2024-11-11T20:55:01,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742261_1437 (size=12847) 2024-11-11T20:55:01,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-11T20:55:01,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:01,868 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 82ee51cab8163979e03e6fa77fa08aaf 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T20:55:01,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=A 2024-11-11T20:55:01,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:01,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=B 2024-11-11T20:55:01,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:01,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=C 2024-11-11T20:55:01,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:01,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/e6602df4486e4eecb95dcbd29718b305 is 50, key is test_row_0/A:col10/1731358500896/Put/seqid=0 2024-11-11T20:55:01,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742262_1438 (size=12301) 2024-11-11T20:55:01,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T20:55:02,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:55:02,049 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:55:02,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358562081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:02,090 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,090 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,090 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:02,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358502059, max timestamp=1731358502088Num uniques 859; 2024-11-11T20:55:02,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358502059, max timestamp=1731358502088Num uniques 859; 2024-11-11T20:55:02,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:55:02,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:55:02,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358502059, max timestamp=1731358502088Num uniques 859; 2024-11-11T20:55:02,090 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:55:02,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:02,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358562134, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:02,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:02,150 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/1854c8b3da85402eb285617e675249b9 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/1854c8b3da85402eb285617e675249b9 2024-11-11T20:55:02,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:02,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358562139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:02,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,153 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:02,153 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82ee51cab8163979e03e6fa77fa08aaf/B of 82ee51cab8163979e03e6fa77fa08aaf into 1854c8b3da85402eb285617e675249b9(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:55:02,153 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:55:02,153 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., storeName=82ee51cab8163979e03e6fa77fa08aaf/B, priority=13, startTime=1731358501731; duration=0sec 2024-11-11T20:55:02,153 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:55:02,153 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82ee51cab8163979e03e6fa77fa08aaf:B 2024-11-11T20:55:02,153 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:55:02,154 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:55:02,154 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 82ee51cab8163979e03e6fa77fa08aaf/C is initiating minor compaction (all files) 2024-11-11T20:55:02,154 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82ee51cab8163979e03e6fa77fa08aaf/C in TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:02,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:02,155 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/2d6b8267c03c41609b42e5c90f428fd5, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/8f501d2ab60947eca1e464fe50584f91, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/35902a21d15b49bba3e242b4790657ed] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp, totalSize=33.8 K 2024-11-11T20:55:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36362 deadline: 1731358562146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:02,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358562149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:02,155 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_0' in 82ee51cab8163979e03e6fa77fa08aaf of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8269 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:55:02,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:02,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,155 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:02,157 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d6b8267c03c41609b42e5c90f428fd5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731358497981 2024-11-11T20:55:02,157 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f501d2ab60947eca1e464fe50584f91, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731358498646 2024-11-11T20:55:02,157 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 35902a21d15b49bba3e242b4790657ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731358500854 2024-11-11T20:55:02,159 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/8ff6d89c71ec42d4b1ac46f9764e1545 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/8ff6d89c71ec42d4b1ac46f9764e1545 2024-11-11T20:55:02,162 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82ee51cab8163979e03e6fa77fa08aaf/A of 82ee51cab8163979e03e6fa77fa08aaf into 8ff6d89c71ec42d4b1ac46f9764e1545(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:55:02,162 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:55:02,162 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., storeName=82ee51cab8163979e03e6fa77fa08aaf/A, priority=13, startTime=1731358501731; duration=0sec 2024-11-11T20:55:02,162 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:02,162 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82ee51cab8163979e03e6fa77fa08aaf:A 2024-11-11T20:55:02,162 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82ee51cab8163979e03e6fa77fa08aaf#C#compaction#369 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:55:02,163 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/bbed8e01cad5463cb38dffc244dc741b is 50, key is test_row_0/C:col10/1731358500875/Put/seqid=0 2024-11-11T20:55:02,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742263_1439 (size=12847) 2024-11-11T20:55:02,202 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:02,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358562196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:02,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,203 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:02,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:02,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358562255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:02,263 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:02,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358562255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:02,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,263 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:02,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:02,275 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/e6602df4486e4eecb95dcbd29718b305 2024-11-11T20:55:02,276 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:02,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358562265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:02,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,276 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:02,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/1180a9b7ca7a4e8cb712dd1477c18330 is 50, key is test_row_0/B:col10/1731358500896/Put/seqid=0 2024-11-11T20:55:02,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742264_1440 (size=12301) 2024-11-11T20:55:02,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T20:55:02,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:02,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358562406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:02,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:02,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:02,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358562465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:02,470 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:02,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358562465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:02,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,470 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:02,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,470 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:02,494 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:02,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358562486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:02,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:02,569 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/bbed8e01cad5463cb38dffc244dc741b as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/bbed8e01cad5463cb38dffc244dc741b 2024-11-11T20:55:02,572 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82ee51cab8163979e03e6fa77fa08aaf/C of 82ee51cab8163979e03e6fa77fa08aaf into bbed8e01cad5463cb38dffc244dc741b(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:55:02,572 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:55:02,572 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., storeName=82ee51cab8163979e03e6fa77fa08aaf/C, priority=13, startTime=1731358501731; duration=0sec 2024-11-11T20:55:02,573 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:02,573 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82ee51cab8163979e03e6fa77fa08aaf:C 2024-11-11T20:55:02,684 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/1180a9b7ca7a4e8cb712dd1477c18330 2024-11-11T20:55:02,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/6a8b290a56184b048eddcae2fa5262f7 is 50, key is test_row_0/C:col10/1731358500896/Put/seqid=0 2024-11-11T20:55:02,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742265_1441 (size=12301) 2024-11-11T20:55:02,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:02,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358562715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:02,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:02,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358562775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:02,779 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:02,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358562775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:02,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:02,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,780 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:02,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:02,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358562806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:02,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:02,814 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:03,093 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/6a8b290a56184b048eddcae2fa5262f7 2024-11-11T20:55:03,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/e6602df4486e4eecb95dcbd29718b305 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/e6602df4486e4eecb95dcbd29718b305 2024-11-11T20:55:03,099 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/e6602df4486e4eecb95dcbd29718b305, entries=150, sequenceid=285, filesize=12.0 K 2024-11-11T20:55:03,100 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/1180a9b7ca7a4e8cb712dd1477c18330 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/1180a9b7ca7a4e8cb712dd1477c18330 2024-11-11T20:55:03,103 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/1180a9b7ca7a4e8cb712dd1477c18330, entries=150, sequenceid=285, filesize=12.0 K 2024-11-11T20:55:03,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/6a8b290a56184b048eddcae2fa5262f7 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/6a8b290a56184b048eddcae2fa5262f7 2024-11-11T20:55:03,111 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/6a8b290a56184b048eddcae2fa5262f7, entries=150, sequenceid=285, filesize=12.0 K 2024-11-11T20:55:03,111 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 82ee51cab8163979e03e6fa77fa08aaf in 1243ms, sequenceid=285, compaction requested=false 2024-11-11T20:55:03,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:55:03,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:03,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-11T20:55:03,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-11T20:55:03,115 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-11T20:55:03,115 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8550 sec 2024-11-11T20:55:03,117 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.8600 sec 2024-11-11T20:55:03,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:55:03,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82ee51cab8163979e03e6fa77fa08aaf 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-11T20:55:03,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=A 2024-11-11T20:55:03,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:55:03,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=B 2024-11-11T20:55:03,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:55:03,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=C 2024-11-11T20:55:03,243 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:55:03,247 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/c6ca39678c5646c5baf9c377fe546422 is 50, key is test_row_0/A:col10/1731358503241/Put/seqid=0 2024-11-11T20:55:03,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742266_1442 (size=14741) 2024-11-11T20:55:03,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:03,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358563293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:03,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:03,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:03,303 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:03,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:03,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358563302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:03,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:03,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:03,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:03,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:03,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358563303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:03,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:03,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:03,321 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:03,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:03,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358563326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:03,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:03,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:03,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:03,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T20:55:03,406 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:55:03,407 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:55:03,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-11T20:55:03,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T20:55:03,408 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:55:03,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:03,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358563406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:03,409 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:55:03,409 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:55:03,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:03,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:03,410 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:03,425 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358563415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:03,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:03,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:03,426 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:03,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358563426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:03,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:03,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:03,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:03,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T20:55:03,562 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T20:55:03,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:03,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:55:03,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:03,562 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:03,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:03,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:03,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:03,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358563615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:03,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:03,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:03,623 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:03,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358563635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:03,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:03,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:03,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:03,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358563646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:03,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/c6ca39678c5646c5baf9c377fe546422 2024-11-11T20:55:03,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:03,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:03,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:03,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/d6e253462b0741e09f165701950caf73 is 50, key is test_row_0/B:col10/1731358503241/Put/seqid=0 2024-11-11T20:55:03,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742267_1443 (size=12301) 2024-11-11T20:55:03,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T20:55:03,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:03,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:55:03,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:03,715 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:03,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:03,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:03,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T20:55:03,867 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T20:55:03,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:03,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:55:03,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:03,868 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:03,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:03,939 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:03,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358563935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:03,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:03,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:03,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:03,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:03,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358563955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:03,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:03,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:03,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:03,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:03,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358563956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:03,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:03,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:03,963 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:04,020 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T20:55:04,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:04,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:55:04,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:04,020 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:04,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:04,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:04,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T20:55:04,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/d6e253462b0741e09f165701950caf73 2024-11-11T20:55:04,065 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/fd1b2c4122f14189becd213a706785e1 is 50, key is test_row_0/C:col10/1731358503241/Put/seqid=0 2024-11-11T20:55:04,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742268_1444 (size=12301) 2024-11-11T20:55:04,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T20:55:04,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:04,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:55:04,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:04,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:04,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:04,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:04,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T20:55:04,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:04,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:55:04,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:04,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:04,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:04,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:04,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:04,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358564345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:04,351 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in 82ee51cab8163979e03e6fa77fa08aaf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:55:04,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:04,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:04,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:04,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:04,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358564445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:04,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:04,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:04,450 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:04,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/fd1b2c4122f14189becd213a706785e1 2024-11-11T20:55:04,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/c6ca39678c5646c5baf9c377fe546422 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/c6ca39678c5646c5baf9c377fe546422 2024-11-11T20:55:04,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/c6ca39678c5646c5baf9c377fe546422, entries=200, sequenceid=324, filesize=14.4 K 2024-11-11T20:55:04,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/d6e253462b0741e09f165701950caf73 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/d6e253462b0741e09f165701950caf73 2024-11-11T20:55:04,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T20:55:04,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:04,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:55:04,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:04,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:04,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:04,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:04,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/d6e253462b0741e09f165701950caf73, entries=150, sequenceid=324, filesize=12.0 K 2024-11-11T20:55:04,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/fd1b2c4122f14189becd213a706785e1 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/fd1b2c4122f14189becd213a706785e1 2024-11-11T20:55:04,483 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:04,483 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/fd1b2c4122f14189becd213a706785e1, entries=150, sequenceid=324, filesize=12.0 K 2024-11-11T20:55:04,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358564476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:04,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:04,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:04,483 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:04,483 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 82ee51cab8163979e03e6fa77fa08aaf in 1241ms, sequenceid=324, compaction requested=true 2024-11-11T20:55:04,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:55:04,483 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82ee51cab8163979e03e6fa77fa08aaf:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:55:04,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:04,484 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:55:04,484 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:55:04,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82ee51cab8163979e03e6fa77fa08aaf:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:55:04,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:04,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82ee51cab8163979e03e6fa77fa08aaf:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:55:04,484 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:55:04,484 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:55:04,484 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:55:04,484 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 82ee51cab8163979e03e6fa77fa08aaf/B is initiating minor compaction (all files) 2024-11-11T20:55:04,484 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 82ee51cab8163979e03e6fa77fa08aaf/A is initiating minor compaction (all files) 2024-11-11T20:55:04,485 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82ee51cab8163979e03e6fa77fa08aaf/A in TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:04,485 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/8ff6d89c71ec42d4b1ac46f9764e1545, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/e6602df4486e4eecb95dcbd29718b305, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/c6ca39678c5646c5baf9c377fe546422] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp, totalSize=39.0 K 2024-11-11T20:55:04,485 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82ee51cab8163979e03e6fa77fa08aaf/B in TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:04,485 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/1854c8b3da85402eb285617e675249b9, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/1180a9b7ca7a4e8cb712dd1477c18330, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/d6e253462b0741e09f165701950caf73] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp, totalSize=36.6 K 2024-11-11T20:55:04,485 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ff6d89c71ec42d4b1ac46f9764e1545, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731358500854 2024-11-11T20:55:04,485 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 1854c8b3da85402eb285617e675249b9, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731358500854 2024-11-11T20:55:04,485 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6602df4486e4eecb95dcbd29718b305, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731358500889 2024-11-11T20:55:04,485 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 1180a9b7ca7a4e8cb712dd1477c18330, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731358500889 2024-11-11T20:55:04,486 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting c6ca39678c5646c5baf9c377fe546422, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731358502133 2024-11-11T20:55:04,486 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting d6e253462b0741e09f165701950caf73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731358502138 2024-11-11T20:55:04,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:55:04,487 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82ee51cab8163979e03e6fa77fa08aaf 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T20:55:04,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=A 2024-11-11T20:55:04,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:04,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=B 2024-11-11T20:55:04,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:04,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=C 2024-11-11T20:55:04,487 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:04,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/d6f31f32e7d04ca38d66f69d769e423d is 50, key is test_row_0/A:col10/1731358503301/Put/seqid=0 2024-11-11T20:55:04,493 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82ee51cab8163979e03e6fa77fa08aaf#A#compaction#376 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:55:04,493 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/1a1b75c7bb8f41b09023a844304e944b is 50, key is test_row_0/A:col10/1731358503241/Put/seqid=0 2024-11-11T20:55:04,495 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82ee51cab8163979e03e6fa77fa08aaf#B#compaction#377 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:55:04,496 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/f2c27f9c9ef846bf87f2d914e3cb569a is 50, key is test_row_0/B:col10/1731358503241/Put/seqid=0 2024-11-11T20:55:04,498 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742269_1445 (size=12301) 2024-11-11T20:55:04,503 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/d6f31f32e7d04ca38d66f69d769e423d 2024-11-11T20:55:04,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742270_1446 (size=12949) 2024-11-11T20:55:04,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742271_1447 (size=12949) 2024-11-11T20:55:04,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/03894b98874548d39edccd51b5a145b9 is 50, key is test_row_0/B:col10/1731358503301/Put/seqid=0 2024-11-11T20:55:04,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742272_1448 (size=12301) 2024-11-11T20:55:04,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T20:55:04,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358504575, max timestamp=1731358504618Num uniques 859; 2024-11-11T20:55:04,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:55:04,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358504575, max timestamp=1731358504618Num uniques 859; 2024-11-11T20:55:04,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358504575, max timestamp=1731358504618Num uniques 859; 2024-11-11T20:55:04,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:55:04,620 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:55:04,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T20:55:04,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:04,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:55:04,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:04,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:04,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:04,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:04,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:04,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358564755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:04,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:04,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:04,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:04,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T20:55:04,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:04,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:55:04,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:04,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:04,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:04,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:04,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:04,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358564876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:04,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:04,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:04,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:04,908 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/1a1b75c7bb8f41b09023a844304e944b as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/1a1b75c7bb8f41b09023a844304e944b 2024-11-11T20:55:04,912 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82ee51cab8163979e03e6fa77fa08aaf/A of 82ee51cab8163979e03e6fa77fa08aaf into 1a1b75c7bb8f41b09023a844304e944b(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:55:04,912 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:55:04,912 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., storeName=82ee51cab8163979e03e6fa77fa08aaf/A, priority=13, startTime=1731358504483; duration=0sec 2024-11-11T20:55:04,912 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:55:04,912 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82ee51cab8163979e03e6fa77fa08aaf:A 2024-11-11T20:55:04,912 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:55:04,913 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:55:04,913 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 82ee51cab8163979e03e6fa77fa08aaf/C is initiating minor compaction (all files) 2024-11-11T20:55:04,913 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82ee51cab8163979e03e6fa77fa08aaf/C in TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:04,913 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/bbed8e01cad5463cb38dffc244dc741b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/6a8b290a56184b048eddcae2fa5262f7, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/fd1b2c4122f14189becd213a706785e1] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp, totalSize=36.6 K 2024-11-11T20:55:04,913 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbed8e01cad5463cb38dffc244dc741b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731358500854 2024-11-11T20:55:04,914 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a8b290a56184b048eddcae2fa5262f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731358500889 2024-11-11T20:55:04,914 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd1b2c4122f14189becd213a706785e1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731358502138 2024-11-11T20:55:04,915 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/f2c27f9c9ef846bf87f2d914e3cb569a as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/f2c27f9c9ef846bf87f2d914e3cb569a 2024-11-11T20:55:04,916 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/03894b98874548d39edccd51b5a145b9 2024-11-11T20:55:04,919 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82ee51cab8163979e03e6fa77fa08aaf/B of 82ee51cab8163979e03e6fa77fa08aaf into f2c27f9c9ef846bf87f2d914e3cb569a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:55:04,920 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:55:04,920 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., storeName=82ee51cab8163979e03e6fa77fa08aaf/B, priority=13, startTime=1731358504484; duration=0sec 2024-11-11T20:55:04,920 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:04,920 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82ee51cab8163979e03e6fa77fa08aaf:B 2024-11-11T20:55:04,920 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82ee51cab8163979e03e6fa77fa08aaf#C#compaction#379 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:55:04,920 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/98caa560dc50450b8b5db8102c590975 is 50, key is test_row_0/C:col10/1731358503241/Put/seqid=0 2024-11-11T20:55:04,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/5ca2489638e8430db654399ac2ba5cbd is 50, key is test_row_0/C:col10/1731358503301/Put/seqid=0 2024-11-11T20:55:04,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742273_1449 (size=12949) 2024-11-11T20:55:04,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742274_1450 (size=12301) 2024-11-11T20:55:04,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/5ca2489638e8430db654399ac2ba5cbd 2024-11-11T20:55:04,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/d6f31f32e7d04ca38d66f69d769e423d as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/d6f31f32e7d04ca38d66f69d769e423d 2024-11-11T20:55:04,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/d6f31f32e7d04ca38d66f69d769e423d, entries=150, sequenceid=335, filesize=12.0 K 2024-11-11T20:55:04,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T20:55:04,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:04,936 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/03894b98874548d39edccd51b5a145b9 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/03894b98874548d39edccd51b5a145b9 2024-11-11T20:55:04,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:55:04,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:04,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:04,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:04,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:04,939 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/03894b98874548d39edccd51b5a145b9, entries=150, sequenceid=335, filesize=12.0 K 2024-11-11T20:55:04,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/5ca2489638e8430db654399ac2ba5cbd as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/5ca2489638e8430db654399ac2ba5cbd 2024-11-11T20:55:04,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/5ca2489638e8430db654399ac2ba5cbd, entries=150, sequenceid=335, filesize=12.0 K 2024-11-11T20:55:04,943 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 82ee51cab8163979e03e6fa77fa08aaf in 457ms, sequenceid=335, compaction requested=false 2024-11-11T20:55:04,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:55:05,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-11T20:55:05,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:05,089 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 82ee51cab8163979e03e6fa77fa08aaf 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-11T20:55:05,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=A 2024-11-11T20:55:05,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:55:05,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=B 2024-11-11T20:55:05,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:55:05,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=C 2024-11-11T20:55:05,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:55:05,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/c1a4cfd2c29a49a180b7636a4fb9d9f9 is 50, key is test_row_0/A:col10/1731358504664/Put/seqid=0 2024-11-11T20:55:05,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742275_1451 (size=12301) 2024-11-11T20:55:05,105 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:55:05,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:55:05,164 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:05,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358565154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:05,164 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:05,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:05,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358565266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:05,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:05,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:05,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:05,329 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/98caa560dc50450b8b5db8102c590975 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/98caa560dc50450b8b5db8102c590975 2024-11-11T20:55:05,332 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 82ee51cab8163979e03e6fa77fa08aaf/C of 82ee51cab8163979e03e6fa77fa08aaf into 98caa560dc50450b8b5db8102c590975(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:55:05,333 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:55:05,333 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., storeName=82ee51cab8163979e03e6fa77fa08aaf/C, priority=13, startTime=1731358504484; duration=0sec 2024-11-11T20:55:05,333 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:05,333 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82ee51cab8163979e03e6fa77fa08aaf:C 2024-11-11T20:55:05,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:05,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358565465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:05,471 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in 82ee51cab8163979e03e6fa77fa08aaf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:55:05,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:05,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:05,472 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:05,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:05,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358565485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:05,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:05,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:05,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:05,497 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/c1a4cfd2c29a49a180b7636a4fb9d9f9 2024-11-11T20:55:05,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:05,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358565496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:05,500 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in 82ee51cab8163979e03e6fa77fa08aaf of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:55:05,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:05,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:05,501 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:05,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/aa5210d0ad2d4c4aa606faba23777f9e is 50, key is test_row_0/B:col10/1731358504664/Put/seqid=0 2024-11-11T20:55:05,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742276_1452 (size=12301) 2024-11-11T20:55:05,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T20:55:05,810 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:05,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358565805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:05,811 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:05,907 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/aa5210d0ad2d4c4aa606faba23777f9e 2024-11-11T20:55:05,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/96f3f04adce144d5abc7cb932144fa0b is 50, key is test_row_0/C:col10/1731358504664/Put/seqid=0 2024-11-11T20:55:05,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742277_1453 (size=12301) 2024-11-11T20:55:06,316 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=372 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/96f3f04adce144d5abc7cb932144fa0b 2024-11-11T20:55:06,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/c1a4cfd2c29a49a180b7636a4fb9d9f9 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/c1a4cfd2c29a49a180b7636a4fb9d9f9 2024-11-11T20:55:06,323 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/c1a4cfd2c29a49a180b7636a4fb9d9f9, entries=150, sequenceid=372, filesize=12.0 K 2024-11-11T20:55:06,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/aa5210d0ad2d4c4aa606faba23777f9e as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/aa5210d0ad2d4c4aa606faba23777f9e 2024-11-11T20:55:06,327 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/aa5210d0ad2d4c4aa606faba23777f9e, entries=150, sequenceid=372, filesize=12.0 K 2024-11-11T20:55:06,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/96f3f04adce144d5abc7cb932144fa0b as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/96f3f04adce144d5abc7cb932144fa0b 2024-11-11T20:55:06,330 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/96f3f04adce144d5abc7cb932144fa0b, entries=150, sequenceid=372, filesize=12.0 K 2024-11-11T20:55:06,331 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 82ee51cab8163979e03e6fa77fa08aaf in 1241ms, sequenceid=372, compaction requested=true 2024-11-11T20:55:06,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:55:06,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:06,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-11T20:55:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-11T20:55:06,333 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-11T20:55:06,333 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9230 sec 2024-11-11T20:55:06,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:55:06,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82ee51cab8163979e03e6fa77fa08aaf 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T20:55:06,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=A 2024-11-11T20:55:06,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:06,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=B 2024-11-11T20:55:06,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:06,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=C 2024-11-11T20:55:06,335 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:06,337 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 2.9270 sec 2024-11-11T20:55:06,339 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/c602dfbce0a5453daf5d3ed4c567de57 is 50, key is test_row_0/A:col10/1731358505153/Put/seqid=0 2024-11-11T20:55:06,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742278_1454 (size=14741) 2024-11-11T20:55:06,425 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:06,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358566424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:06,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358506399, max timestamp=1731358506423Num uniques 859; 2024-11-11T20:55:06,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358506399, max timestamp=1731358506423Num uniques 859; 2024-11-11T20:55:06,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731358506399, max timestamp=1731358506423Num uniques 859; 2024-11-11T20:55:06,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:55:06,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:55:06,426 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/51ca66f7ee3c:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T20:55:06,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:06,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:06,427 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:06,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:06,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358566536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:06,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:06,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:06,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:06,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:06,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358566537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:06,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:06,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:06,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:06,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:06,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358566656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:06,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:06,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:06,663 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:06,744 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/c602dfbce0a5453daf5d3ed4c567de57 2024-11-11T20:55:06,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/3714ceb2885d46fc9684d7a7c3470ecd is 50, key is test_row_0/B:col10/1731358505153/Put/seqid=0 2024-11-11T20:55:06,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742279_1455 (size=12301) 2024-11-11T20:55:06,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:06,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358566756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:06,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:06,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:06,764 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:06,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:06,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358566866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:06,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:06,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:06,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:07,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:07,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358567075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:07,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:07,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:07,083 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:07,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/3714ceb2885d46fc9684d7a7c3470ecd 2024-11-11T20:55:07,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/49ce0c27fcd64f9eaa443ee41662b902 is 50, key is test_row_0/C:col10/1731358505153/Put/seqid=0 2024-11-11T20:55:07,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742280_1456 (size=12301) 2024-11-11T20:55:07,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:07,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358567175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:07,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:07,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:07,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:07,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:07,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36408 deadline: 1731358567496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:07,502 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in 82ee51cab8163979e03e6fa77fa08aaf of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:55:07,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:07,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:07,502 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:07,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:07,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36342 deadline: 1731358567506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:07,514 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in 82ee51cab8163979e03e6fa77fa08aaf of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:55:07,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:07,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:07,515 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:07,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/49ce0c27fcd64f9eaa443ee41662b902 2024-11-11T20:55:07,575 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/c602dfbce0a5453daf5d3ed4c567de57 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/c602dfbce0a5453daf5d3ed4c567de57 2024-11-11T20:55:07,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/c602dfbce0a5453daf5d3ed4c567de57, entries=200, sequenceid=385, filesize=14.4 K 2024-11-11T20:55:07,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/3714ceb2885d46fc9684d7a7c3470ecd as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/3714ceb2885d46fc9684d7a7c3470ecd 2024-11-11T20:55:07,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/3714ceb2885d46fc9684d7a7c3470ecd, entries=150, sequenceid=385, filesize=12.0 K 2024-11-11T20:55:07,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/49ce0c27fcd64f9eaa443ee41662b902 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/49ce0c27fcd64f9eaa443ee41662b902 2024-11-11T20:55:07,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/49ce0c27fcd64f9eaa443ee41662b902, entries=150, sequenceid=385, filesize=12.0 K 2024-11-11T20:55:07,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=120.76 KB/123660 for 82ee51cab8163979e03e6fa77fa08aaf in 1249ms, sequenceid=385, compaction requested=true 2024-11-11T20:55:07,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:55:07,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82ee51cab8163979e03e6fa77fa08aaf:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:55:07,585 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:55:07,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:07,585 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:55:07,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82ee51cab8163979e03e6fa77fa08aaf:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:55:07,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:07,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 82ee51cab8163979e03e6fa77fa08aaf:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:55:07,585 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:55:07,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-11T20:55:07,586 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:55:07,586 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:55:07,587 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52292 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:55:07,587 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:55:07,587 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): 82ee51cab8163979e03e6fa77fa08aaf/A is initiating minor compaction (all files) 2024-11-11T20:55:07,587 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 82ee51cab8163979e03e6fa77fa08aaf/B is initiating minor compaction (all files) 2024-11-11T20:55:07,587 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82ee51cab8163979e03e6fa77fa08aaf/A in TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:07,587 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82ee51cab8163979e03e6fa77fa08aaf/B in TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:07,587 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/f2c27f9c9ef846bf87f2d914e3cb569a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/03894b98874548d39edccd51b5a145b9, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/aa5210d0ad2d4c4aa606faba23777f9e, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/3714ceb2885d46fc9684d7a7c3470ecd] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp, totalSize=48.7 K 2024-11-11T20:55:07,587 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/1a1b75c7bb8f41b09023a844304e944b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/d6f31f32e7d04ca38d66f69d769e423d, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/c1a4cfd2c29a49a180b7636a4fb9d9f9, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/c602dfbce0a5453daf5d3ed4c567de57] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp, totalSize=51.1 K 2024-11-11T20:55:07,587 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting f2c27f9c9ef846bf87f2d914e3cb569a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731358502138 2024-11-11T20:55:07,587 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a1b75c7bb8f41b09023a844304e944b, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731358502138 2024-11-11T20:55:07,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-11T20:55:07,587 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 03894b98874548d39edccd51b5a145b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731358503291 2024-11-11T20:55:07,587 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6f31f32e7d04ca38d66f69d769e423d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731358503291 2024-11-11T20:55:07,588 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:55:07,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T20:55:07,588 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:55:07,588 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:55:07,588 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting aa5210d0ad2d4c4aa606faba23777f9e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731358504664 2024-11-11T20:55:07,588 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1a4cfd2c29a49a180b7636a4fb9d9f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731358504664 2024-11-11T20:55:07,589 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 3714ceb2885d46fc9684d7a7c3470ecd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731358505145 2024-11-11T20:55:07,589 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting c602dfbce0a5453daf5d3ed4c567de57, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731358505140 2024-11-11T20:55:07,595 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82ee51cab8163979e03e6fa77fa08aaf#B#compaction#387 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:55:07,596 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/3638b9259e364792b25513b3e3d3343c is 50, key is test_row_0/B:col10/1731358505153/Put/seqid=0 2024-11-11T20:55:07,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742281_1457 (size=13085) 2024-11-11T20:55:07,599 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82ee51cab8163979e03e6fa77fa08aaf#A#compaction#388 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:55:07,600 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/1fe192569a7944879d82c4102ea48247 is 50, key is test_row_0/A:col10/1731358505153/Put/seqid=0 2024-11-11T20:55:07,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:55:07,605 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 82ee51cab8163979e03e6fa77fa08aaf 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-11T20:55:07,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=A 2024-11-11T20:55:07,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:55:07,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=B 2024-11-11T20:55:07,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:55:07,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=C 2024-11-11T20:55:07,605 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:55:07,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742282_1458 (size=13085) 2024-11-11T20:55:07,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/5db9b63fecc74fd899e85392ae1679b9 is 50, key is test_row_0/A:col10/1731358506509/Put/seqid=0 2024-11-11T20:55:07,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742283_1459 (size=14741) 2024-11-11T20:55:07,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:07,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358567668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:07,677 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:07,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T20:55:07,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:07,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36354 deadline: 1731358567696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:07,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:07,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T20:55:07,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:07,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:55:07,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:07,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:07,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:07,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:07,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:07,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358567786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:07,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:07,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:07,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:07,892 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T20:55:07,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:07,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:55:07,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:07,892 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:07,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:07,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:07,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T20:55:08,002 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/3638b9259e364792b25513b3e3d3343c as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/3638b9259e364792b25513b3e3d3343c 2024-11-11T20:55:08,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:08,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358567995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:08,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:08,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:08,004 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:08,005 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 82ee51cab8163979e03e6fa77fa08aaf/B of 82ee51cab8163979e03e6fa77fa08aaf into 3638b9259e364792b25513b3e3d3343c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:55:08,005 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:55:08,005 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., storeName=82ee51cab8163979e03e6fa77fa08aaf/B, priority=12, startTime=1731358507585; duration=0sec 2024-11-11T20:55:08,005 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:55:08,005 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82ee51cab8163979e03e6fa77fa08aaf:B 2024-11-11T20:55:08,006 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:55:08,006 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49852 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:55:08,006 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): 82ee51cab8163979e03e6fa77fa08aaf/C is initiating minor compaction (all files) 2024-11-11T20:55:08,006 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 82ee51cab8163979e03e6fa77fa08aaf/C in TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:08,007 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/98caa560dc50450b8b5db8102c590975, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/5ca2489638e8430db654399ac2ba5cbd, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/96f3f04adce144d5abc7cb932144fa0b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/49ce0c27fcd64f9eaa443ee41662b902] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp, totalSize=48.7 K 2024-11-11T20:55:08,007 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 98caa560dc50450b8b5db8102c590975, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731358502138 2024-11-11T20:55:08,007 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ca2489638e8430db654399ac2ba5cbd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731358503291 2024-11-11T20:55:08,007 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 96f3f04adce144d5abc7cb932144fa0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=372, earliestPutTs=1731358504664 2024-11-11T20:55:08,007 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 49ce0c27fcd64f9eaa443ee41662b902, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731358505145 2024-11-11T20:55:08,010 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/1fe192569a7944879d82c4102ea48247 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/1fe192569a7944879d82c4102ea48247 2024-11-11T20:55:08,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/5db9b63fecc74fd899e85392ae1679b9 2024-11-11T20:55:08,015 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 82ee51cab8163979e03e6fa77fa08aaf#C#compaction#390 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:55:08,015 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/735e9be7e02a4f709abdab39f641a5da is 50, key is test_row_0/C:col10/1731358505153/Put/seqid=0 2024-11-11T20:55:08,016 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 82ee51cab8163979e03e6fa77fa08aaf/A of 82ee51cab8163979e03e6fa77fa08aaf into 1fe192569a7944879d82c4102ea48247(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:55:08,016 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:55:08,016 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., storeName=82ee51cab8163979e03e6fa77fa08aaf/A, priority=12, startTime=1731358507585; duration=0sec 2024-11-11T20:55:08,017 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:08,017 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82ee51cab8163979e03e6fa77fa08aaf:A 2024-11-11T20:55:08,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/6a3305c9d66b452fa3f03b7d390d752b is 50, key is test_row_0/B:col10/1731358506509/Put/seqid=0 2024-11-11T20:55:08,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742284_1460 (size=13085) 2024-11-11T20:55:08,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742285_1461 (size=12301) 2024-11-11T20:55:08,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/6a3305c9d66b452fa3f03b7d390d752b 2024-11-11T20:55:08,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/ac2a55b0d71a49938e88de6f2730a14f is 50, key is test_row_0/C:col10/1731358506509/Put/seqid=0 2024-11-11T20:55:08,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742286_1462 (size=12301) 2024-11-11T20:55:08,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T20:55:08,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:08,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:55:08,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:08,045 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:08,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:08,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:08,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T20:55:08,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:08,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:55:08,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:08,198 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:08,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:08,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:08,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T20:55:08,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:08,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36332 deadline: 1731358568315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:08,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 , the old value is region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:08,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:08,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=2 because the exception is null or not the one we care about 2024-11-11T20:55:08,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T20:55:08,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:08,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:55:08,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:08,350 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:08,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:08,430 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/735e9be7e02a4f709abdab39f641a5da as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/735e9be7e02a4f709abdab39f641a5da 2024-11-11T20:55:08,433 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 82ee51cab8163979e03e6fa77fa08aaf/C of 82ee51cab8163979e03e6fa77fa08aaf into 735e9be7e02a4f709abdab39f641a5da(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:55:08,433 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:55:08,433 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., storeName=82ee51cab8163979e03e6fa77fa08aaf/C, priority=12, startTime=1731358507585; duration=0sec 2024-11-11T20:55:08,433 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:08,433 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 82ee51cab8163979e03e6fa77fa08aaf:C 2024-11-11T20:55:08,438 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=420 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/ac2a55b0d71a49938e88de6f2730a14f 2024-11-11T20:55:08,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/5db9b63fecc74fd899e85392ae1679b9 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/5db9b63fecc74fd899e85392ae1679b9 2024-11-11T20:55:08,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/5db9b63fecc74fd899e85392ae1679b9, entries=200, sequenceid=420, filesize=14.4 K 2024-11-11T20:55:08,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/6a3305c9d66b452fa3f03b7d390d752b as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/6a3305c9d66b452fa3f03b7d390d752b 2024-11-11T20:55:08,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/6a3305c9d66b452fa3f03b7d390d752b, entries=150, sequenceid=420, filesize=12.0 K 2024-11-11T20:55:08,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/ac2a55b0d71a49938e88de6f2730a14f as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/ac2a55b0d71a49938e88de6f2730a14f 2024-11-11T20:55:08,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/ac2a55b0d71a49938e88de6f2730a14f, entries=150, sequenceid=420, filesize=12.0 K 2024-11-11T20:55:08,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 82ee51cab8163979e03e6fa77fa08aaf in 847ms, sequenceid=420, compaction requested=false 2024-11-11T20:55:08,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:55:08,475 INFO [Thread-1780 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1780. 2024-11-11T20:55:08,475 DEBUG [Thread-1780 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:55:08,475 DEBUG [Thread-1780 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:08,475 DEBUG [Thread-1780 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:08,476 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:55:08,478 INFO [Thread-1784 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1784. 2024-11-11T20:55:08,479 DEBUG [Thread-1784 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:55:08,479 DEBUG [Thread-1784 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:08,479 DEBUG [Thread-1784 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:08,479 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:55:08,479 INFO [Thread-1786 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1786. 2024-11-11T20:55:08,479 DEBUG [Thread-1786 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:55:08,479 DEBUG [Thread-1786 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:08,480 DEBUG [Thread-1786 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:08,480 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:55:08,481 INFO [Thread-1778 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1778. 2024-11-11T20:55:08,481 DEBUG [Thread-1778 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:55:08,481 DEBUG [Thread-1778 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:08,481 DEBUG [Thread-1778 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:08,481 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:55:08,483 INFO [Thread-1782 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1782. 2024-11-11T20:55:08,483 DEBUG [Thread-1782 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:55:08,483 DEBUG [Thread-1782 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:08,483 DEBUG [Thread-1782 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:08,483 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:55:08,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-11T20:55:08,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:08,503 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 82ee51cab8163979e03e6fa77fa08aaf 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T20:55:08,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=A 2024-11-11T20:55:08,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:08,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=B 2024-11-11T20:55:08,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:08,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=C 2024-11-11T20:55:08,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:08,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/cb155b95e34a476e87f1b6460fc888f5 is 50, key is test_row_0/A:col10/1731358507651/Put/seqid=0 2024-11-11T20:55:08,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742287_1463 (size=12301) 2024-11-11T20:55:08,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:55:08,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. as already flushing 2024-11-11T20:55:08,720 INFO [Thread-1773 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1773. 2024-11-11T20:55:08,720 DEBUG [Thread-1773 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:55:08,720 DEBUG [Thread-1773 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:08,720 DEBUG [Thread-1773 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:08,721 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:55:08,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T20:55:08,830 INFO [Thread-1767 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1767. 2024-11-11T20:55:08,830 DEBUG [Thread-1767 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:55:08,830 DEBUG [Thread-1767 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:08,831 DEBUG [Thread-1767 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:08,831 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:55:08,915 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/cb155b95e34a476e87f1b6460fc888f5 2024-11-11T20:55:08,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/89bd8ccb63aa444a90cb013b3012f8d5 is 50, key is test_row_0/B:col10/1731358507651/Put/seqid=0 2024-11-11T20:55:08,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742288_1464 (size=12301) 2024-11-11T20:55:09,333 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/89bd8ccb63aa444a90cb013b3012f8d5 2024-11-11T20:55:09,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/bec7ccb923824691839aeaf8ead15a31 is 50, key is test_row_0/C:col10/1731358507651/Put/seqid=0 2024-11-11T20:55:09,349 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742289_1465 (size=12301) 2024-11-11T20:55:09,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T20:55:09,751 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/bec7ccb923824691839aeaf8ead15a31 2024-11-11T20:55:09,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/cb155b95e34a476e87f1b6460fc888f5 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/cb155b95e34a476e87f1b6460fc888f5 2024-11-11T20:55:09,767 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/cb155b95e34a476e87f1b6460fc888f5, entries=150, sequenceid=434, filesize=12.0 K 2024-11-11T20:55:09,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/89bd8ccb63aa444a90cb013b3012f8d5 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/89bd8ccb63aa444a90cb013b3012f8d5 2024-11-11T20:55:09,771 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/89bd8ccb63aa444a90cb013b3012f8d5, entries=150, sequenceid=434, filesize=12.0 K 2024-11-11T20:55:09,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/bec7ccb923824691839aeaf8ead15a31 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/bec7ccb923824691839aeaf8ead15a31 2024-11-11T20:55:09,775 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/bec7ccb923824691839aeaf8ead15a31, entries=150, sequenceid=434, filesize=12.0 K 2024-11-11T20:55:09,776 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=13.42 KB/13740 for 82ee51cab8163979e03e6fa77fa08aaf in 1273ms, sequenceid=434, compaction requested=true 2024-11-11T20:55:09,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 82ee51cab8163979e03e6fa77fa08aaf: 2024-11-11T20:55:09,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:09,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-11T20:55:09,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-11T20:55:09,780 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-11T20:55:09,780 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1900 sec 2024-11-11T20:55:09,782 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 2.1950 sec 2024-11-11T20:55:10,578 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T20:55:11,530 INFO [Thread-1771 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1771. 2024-11-11T20:55:11,531 DEBUG [Thread-1771 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:55:11,531 DEBUG [Thread-1771 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:11,531 DEBUG [Thread-1771 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:11,531 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:55:11,559 INFO [Thread-1775 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1775. 2024-11-11T20:55:11,559 DEBUG [Thread-1775 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:55:11,559 DEBUG [Thread-1775 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:11,560 DEBUG [Thread-1775 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:11,560 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:55:11,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T20:55:11,767 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:55:12,200 INFO [Thread-1769 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1769. 2024-11-11T20:55:12,200 DEBUG [Thread-1769 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:55:12,200 DEBUG [Thread-1769 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:12,201 DEBUG [Thread-1769 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:12,201 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:55:12,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T20:55:12,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 114 2024-11-11T20:55:12,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 23 2024-11-11T20:55:12,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 64 2024-11-11T20:55:12,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-11T20:55:12,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-11T20:55:12,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T20:55:12,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T20:55:12,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1296 2024-11-11T20:55:12,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3888 rows 2024-11-11T20:55:12,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1290 2024-11-11T20:55:12,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3870 rows 2024-11-11T20:55:12,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1297 2024-11-11T20:55:12,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3891 rows 2024-11-11T20:55:12,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1298 2024-11-11T20:55:12,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3894 rows 2024-11-11T20:55:12,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1300 2024-11-11T20:55:12,202 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3900 rows 2024-11-11T20:55:12,203 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T20:55:12,203 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:55:12,203 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:12,204 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:12,204 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:55:12,205 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T20:55:12,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T20:55:12,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T20:55:12,210 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358512210"}]},"ts":"1731358512210"} 2024-11-11T20:55:12,212 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T20:55:12,212 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T20:55:12,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T20:55:12,214 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=82ee51cab8163979e03e6fa77fa08aaf, UNASSIGN}] 2024-11-11T20:55:12,215 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=82ee51cab8163979e03e6fa77fa08aaf, UNASSIGN 2024-11-11T20:55:12,216 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=82ee51cab8163979e03e6fa77fa08aaf, regionState=CLOSING, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:12,218 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=125, ppid=124, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=82ee51cab8163979e03e6fa77fa08aaf, UNASSIGN because future has completed 2024-11-11T20:55:12,218 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T20:55:12,218 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; CloseRegionProcedure 82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066}] 2024-11-11T20:55:12,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T20:55:12,371 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(122): Close 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:55:12,371 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T20:55:12,372 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1722): Closing 82ee51cab8163979e03e6fa77fa08aaf, disabling compactions & flushes 2024-11-11T20:55:12,372 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:12,372 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:12,372 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. after waiting 0 ms 2024-11-11T20:55:12,372 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:12,372 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(2902): Flushing 82ee51cab8163979e03e6fa77fa08aaf 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T20:55:12,373 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=A 2024-11-11T20:55:12,373 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:12,373 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=B 2024-11-11T20:55:12,373 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:12,373 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 82ee51cab8163979e03e6fa77fa08aaf, store=C 2024-11-11T20:55:12,373 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:12,381 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/18c740f82ff5418b8884a78664553c00 is 50, key is test_row_0/A:col10/1731358512197/Put/seqid=0 2024-11-11T20:55:12,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742290_1466 (size=9857) 2024-11-11T20:55:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T20:55:12,786 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/18c740f82ff5418b8884a78664553c00 2024-11-11T20:55:12,797 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/0f7f0ff2afcb4c93b1193cd930d7d72a is 50, key is test_row_0/B:col10/1731358512197/Put/seqid=0 2024-11-11T20:55:12,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742291_1467 (size=9857) 2024-11-11T20:55:12,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T20:55:13,203 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/0f7f0ff2afcb4c93b1193cd930d7d72a 2024-11-11T20:55:13,215 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/f59274fca5604bf18e73cd866efa648d is 50, key is test_row_0/C:col10/1731358512197/Put/seqid=0 2024-11-11T20:55:13,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742292_1468 (size=9857) 2024-11-11T20:55:13,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T20:55:13,620 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=442 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/f59274fca5604bf18e73cd866efa648d 2024-11-11T20:55:13,631 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/A/18c740f82ff5418b8884a78664553c00 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/18c740f82ff5418b8884a78664553c00 2024-11-11T20:55:13,637 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/18c740f82ff5418b8884a78664553c00, entries=100, sequenceid=442, filesize=9.6 K 2024-11-11T20:55:13,638 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/B/0f7f0ff2afcb4c93b1193cd930d7d72a as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/0f7f0ff2afcb4c93b1193cd930d7d72a 2024-11-11T20:55:13,643 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/0f7f0ff2afcb4c93b1193cd930d7d72a, entries=100, sequenceid=442, filesize=9.6 K 2024-11-11T20:55:13,643 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/.tmp/C/f59274fca5604bf18e73cd866efa648d as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/f59274fca5604bf18e73cd866efa648d 2024-11-11T20:55:13,648 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/f59274fca5604bf18e73cd866efa648d, entries=100, sequenceid=442, filesize=9.6 K 2024-11-11T20:55:13,649 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 82ee51cab8163979e03e6fa77fa08aaf in 1277ms, sequenceid=442, compaction requested=true 2024-11-11T20:55:13,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/3d8380b8e9bd4f7c97b112eb4beef50d, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/0cf4613ac22b4484937b7ed6c5d0e5f5, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/c88566ac1b434f9eb315eb67e68164af, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/b3bdf427bccb4409a1d13dbee6c1294f, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/69e6c63fce4245558245ed3b776807f4, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/715f0e36bbbb4a35846bfc649c6e8950, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/ea713673f21b415ca536e494f2e6b038, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/f7b40cf99573458bb6c8c813e01f0e95, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/90adf1a23ce048eeb6aa808f3783eee5, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/cfcb1f8b4c444fa1a797cdaef792b2bd, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/2eabdd223d734730b7bf60f177779125, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/0c2bdd244a7f45beb59ac6f53ce65921, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/291aae84d7d14a5da0bec946598c9b81, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/ae323a0ddca347a0a67aa40f63357913, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/b30ae60e532042e29e73636dd64fe114, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/9438f70ecc654900b137269714159dcc, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/8ff6d89c71ec42d4b1ac46f9764e1545, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/e6602df4486e4eecb95dcbd29718b305, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/c6ca39678c5646c5baf9c377fe546422, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/1a1b75c7bb8f41b09023a844304e944b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/d6f31f32e7d04ca38d66f69d769e423d, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/c1a4cfd2c29a49a180b7636a4fb9d9f9, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/c602dfbce0a5453daf5d3ed4c567de57] to archive 2024-11-11T20:55:13,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:55:13,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/3d8380b8e9bd4f7c97b112eb4beef50d to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/3d8380b8e9bd4f7c97b112eb4beef50d 2024-11-11T20:55:13,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/0cf4613ac22b4484937b7ed6c5d0e5f5 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/0cf4613ac22b4484937b7ed6c5d0e5f5 2024-11-11T20:55:13,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/c88566ac1b434f9eb315eb67e68164af to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/c88566ac1b434f9eb315eb67e68164af 2024-11-11T20:55:13,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/b3bdf427bccb4409a1d13dbee6c1294f to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/b3bdf427bccb4409a1d13dbee6c1294f 2024-11-11T20:55:13,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/69e6c63fce4245558245ed3b776807f4 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/69e6c63fce4245558245ed3b776807f4 2024-11-11T20:55:13,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/715f0e36bbbb4a35846bfc649c6e8950 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/715f0e36bbbb4a35846bfc649c6e8950 2024-11-11T20:55:13,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/ea713673f21b415ca536e494f2e6b038 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/ea713673f21b415ca536e494f2e6b038 2024-11-11T20:55:13,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/f7b40cf99573458bb6c8c813e01f0e95 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/f7b40cf99573458bb6c8c813e01f0e95 2024-11-11T20:55:13,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/90adf1a23ce048eeb6aa808f3783eee5 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/90adf1a23ce048eeb6aa808f3783eee5 2024-11-11T20:55:13,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/cfcb1f8b4c444fa1a797cdaef792b2bd to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/cfcb1f8b4c444fa1a797cdaef792b2bd 2024-11-11T20:55:13,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/2eabdd223d734730b7bf60f177779125 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/2eabdd223d734730b7bf60f177779125 2024-11-11T20:55:13,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/0c2bdd244a7f45beb59ac6f53ce65921 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/0c2bdd244a7f45beb59ac6f53ce65921 2024-11-11T20:55:13,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/291aae84d7d14a5da0bec946598c9b81 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/291aae84d7d14a5da0bec946598c9b81 2024-11-11T20:55:13,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/ae323a0ddca347a0a67aa40f63357913 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/ae323a0ddca347a0a67aa40f63357913 2024-11-11T20:55:13,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/b30ae60e532042e29e73636dd64fe114 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/b30ae60e532042e29e73636dd64fe114 2024-11-11T20:55:13,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/9438f70ecc654900b137269714159dcc to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/9438f70ecc654900b137269714159dcc 2024-11-11T20:55:13,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/8ff6d89c71ec42d4b1ac46f9764e1545 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/8ff6d89c71ec42d4b1ac46f9764e1545 2024-11-11T20:55:13,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/e6602df4486e4eecb95dcbd29718b305 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/e6602df4486e4eecb95dcbd29718b305 2024-11-11T20:55:13,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/c6ca39678c5646c5baf9c377fe546422 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/c6ca39678c5646c5baf9c377fe546422 2024-11-11T20:55:13,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/1a1b75c7bb8f41b09023a844304e944b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/1a1b75c7bb8f41b09023a844304e944b 2024-11-11T20:55:13,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/d6f31f32e7d04ca38d66f69d769e423d to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/d6f31f32e7d04ca38d66f69d769e423d 2024-11-11T20:55:13,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/c1a4cfd2c29a49a180b7636a4fb9d9f9 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/c1a4cfd2c29a49a180b7636a4fb9d9f9 2024-11-11T20:55:13,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/c602dfbce0a5453daf5d3ed4c567de57 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/c602dfbce0a5453daf5d3ed4c567de57 2024-11-11T20:55:13,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/4807f26a508e420bb4a6da9eaeb048fc, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/d14b6faa464b4cf4987067984f6ee38e, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/2fed4524da744f88b5bb44d572a4c93b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/0a6c2cdf670b44aeb9a872caa434bf05, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/99bd126b8eb8437f8bc59c6c8c29c315, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/f2bfd44d96094657922c5103f1f01584, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/c50aceec6c6340358aa47d5a8fde36d3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/c4a99ff082524989aeb4d1b72bab85d7, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/eaf83a15e1b4405ca458802b3d953262, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/85a4dc4b7a9748cc9b078cc180f4a859, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/720fbaaebb52455fa26464a4572feae3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/a05f338268fb44dabe5c1ad344b0fbc5, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/baaf610561ad4b03afa4ccd7b8a85b85, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/3b755b6c11664405ab40ba57737e5680, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/51a0f4264c224129a665a2c935659fa7, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/1854c8b3da85402eb285617e675249b9, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/e00e25358a8f4a1f931c231477f21b5c, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/1180a9b7ca7a4e8cb712dd1477c18330, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/f2c27f9c9ef846bf87f2d914e3cb569a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/d6e253462b0741e09f165701950caf73, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/03894b98874548d39edccd51b5a145b9, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/aa5210d0ad2d4c4aa606faba23777f9e, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/3714ceb2885d46fc9684d7a7c3470ecd] to archive 2024-11-11T20:55:13,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:55:13,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/4807f26a508e420bb4a6da9eaeb048fc to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/4807f26a508e420bb4a6da9eaeb048fc 2024-11-11T20:55:13,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/d14b6faa464b4cf4987067984f6ee38e to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/d14b6faa464b4cf4987067984f6ee38e 2024-11-11T20:55:13,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/2fed4524da744f88b5bb44d572a4c93b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/2fed4524da744f88b5bb44d572a4c93b 2024-11-11T20:55:13,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/0a6c2cdf670b44aeb9a872caa434bf05 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/0a6c2cdf670b44aeb9a872caa434bf05 2024-11-11T20:55:13,701 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/99bd126b8eb8437f8bc59c6c8c29c315 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/99bd126b8eb8437f8bc59c6c8c29c315 2024-11-11T20:55:13,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/f2bfd44d96094657922c5103f1f01584 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/f2bfd44d96094657922c5103f1f01584 2024-11-11T20:55:13,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/c50aceec6c6340358aa47d5a8fde36d3 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/c50aceec6c6340358aa47d5a8fde36d3 2024-11-11T20:55:13,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/c4a99ff082524989aeb4d1b72bab85d7 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/c4a99ff082524989aeb4d1b72bab85d7 2024-11-11T20:55:13,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/eaf83a15e1b4405ca458802b3d953262 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/eaf83a15e1b4405ca458802b3d953262 2024-11-11T20:55:13,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/85a4dc4b7a9748cc9b078cc180f4a859 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/85a4dc4b7a9748cc9b078cc180f4a859 2024-11-11T20:55:13,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/720fbaaebb52455fa26464a4572feae3 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/720fbaaebb52455fa26464a4572feae3 2024-11-11T20:55:13,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/a05f338268fb44dabe5c1ad344b0fbc5 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/a05f338268fb44dabe5c1ad344b0fbc5 2024-11-11T20:55:13,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/baaf610561ad4b03afa4ccd7b8a85b85 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/baaf610561ad4b03afa4ccd7b8a85b85 2024-11-11T20:55:13,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/3b755b6c11664405ab40ba57737e5680 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/3b755b6c11664405ab40ba57737e5680 2024-11-11T20:55:13,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/51a0f4264c224129a665a2c935659fa7 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/51a0f4264c224129a665a2c935659fa7 2024-11-11T20:55:13,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/1854c8b3da85402eb285617e675249b9 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/1854c8b3da85402eb285617e675249b9 2024-11-11T20:55:13,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/e00e25358a8f4a1f931c231477f21b5c to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/e00e25358a8f4a1f931c231477f21b5c 2024-11-11T20:55:13,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/1180a9b7ca7a4e8cb712dd1477c18330 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/1180a9b7ca7a4e8cb712dd1477c18330 2024-11-11T20:55:13,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/f2c27f9c9ef846bf87f2d914e3cb569a to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/f2c27f9c9ef846bf87f2d914e3cb569a 2024-11-11T20:55:13,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/d6e253462b0741e09f165701950caf73 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/d6e253462b0741e09f165701950caf73 2024-11-11T20:55:13,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/03894b98874548d39edccd51b5a145b9 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/03894b98874548d39edccd51b5a145b9 2024-11-11T20:55:13,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/aa5210d0ad2d4c4aa606faba23777f9e to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/aa5210d0ad2d4c4aa606faba23777f9e 2024-11-11T20:55:13,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/3714ceb2885d46fc9684d7a7c3470ecd to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/3714ceb2885d46fc9684d7a7c3470ecd 2024-11-11T20:55:13,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/5b53510121a44ce08409a48bdd04ffba, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/92bf723a400748f9ad42a66bea804870, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/3479b00fad494f57bf5da26f16517faa, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/c782cd80db514fa0b295f89b3137fc2b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/4fe3eb017baa4fe5b867392716b96f24, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/71c78609c2f2414181c2fd308010838b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/4845dc028557453b81458a76448b91bd, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/5100216d31f345b78e61edafda1e550d, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/15f1dc900dee46b583a99de2997d27a9, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/529822b96634434aa9778ab3c44e9e66, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/93f4dfcaa53e4a0286e8299ec947ca3d, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/b1c7b6a430f840719d3a08793a2bfda2, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/2d6b8267c03c41609b42e5c90f428fd5, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/0ebb40669bec4c9fb1679d6af85ee004, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/8f501d2ab60947eca1e464fe50584f91, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/bbed8e01cad5463cb38dffc244dc741b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/35902a21d15b49bba3e242b4790657ed, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/6a8b290a56184b048eddcae2fa5262f7, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/98caa560dc50450b8b5db8102c590975, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/fd1b2c4122f14189becd213a706785e1, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/5ca2489638e8430db654399ac2ba5cbd, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/96f3f04adce144d5abc7cb932144fa0b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/49ce0c27fcd64f9eaa443ee41662b902] to archive 2024-11-11T20:55:13,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:55:13,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/5b53510121a44ce08409a48bdd04ffba to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/5b53510121a44ce08409a48bdd04ffba 2024-11-11T20:55:13,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/92bf723a400748f9ad42a66bea804870 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/92bf723a400748f9ad42a66bea804870 2024-11-11T20:55:13,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/3479b00fad494f57bf5da26f16517faa to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/3479b00fad494f57bf5da26f16517faa 2024-11-11T20:55:13,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/c782cd80db514fa0b295f89b3137fc2b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/c782cd80db514fa0b295f89b3137fc2b 2024-11-11T20:55:13,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/4fe3eb017baa4fe5b867392716b96f24 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/4fe3eb017baa4fe5b867392716b96f24 2024-11-11T20:55:13,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/71c78609c2f2414181c2fd308010838b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/71c78609c2f2414181c2fd308010838b 2024-11-11T20:55:13,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/4845dc028557453b81458a76448b91bd to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/4845dc028557453b81458a76448b91bd 2024-11-11T20:55:13,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/5100216d31f345b78e61edafda1e550d to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/5100216d31f345b78e61edafda1e550d 2024-11-11T20:55:13,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/15f1dc900dee46b583a99de2997d27a9 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/15f1dc900dee46b583a99de2997d27a9 2024-11-11T20:55:13,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/529822b96634434aa9778ab3c44e9e66 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/529822b96634434aa9778ab3c44e9e66 2024-11-11T20:55:13,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/93f4dfcaa53e4a0286e8299ec947ca3d to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/93f4dfcaa53e4a0286e8299ec947ca3d 2024-11-11T20:55:13,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/b1c7b6a430f840719d3a08793a2bfda2 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/b1c7b6a430f840719d3a08793a2bfda2 2024-11-11T20:55:13,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/2d6b8267c03c41609b42e5c90f428fd5 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/2d6b8267c03c41609b42e5c90f428fd5 2024-11-11T20:55:13,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/0ebb40669bec4c9fb1679d6af85ee004 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/0ebb40669bec4c9fb1679d6af85ee004 2024-11-11T20:55:13,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/8f501d2ab60947eca1e464fe50584f91 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/8f501d2ab60947eca1e464fe50584f91 2024-11-11T20:55:13,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/bbed8e01cad5463cb38dffc244dc741b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/bbed8e01cad5463cb38dffc244dc741b 2024-11-11T20:55:13,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/35902a21d15b49bba3e242b4790657ed to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/35902a21d15b49bba3e242b4790657ed 2024-11-11T20:55:13,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/6a8b290a56184b048eddcae2fa5262f7 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/6a8b290a56184b048eddcae2fa5262f7 2024-11-11T20:55:13,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/98caa560dc50450b8b5db8102c590975 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/98caa560dc50450b8b5db8102c590975 2024-11-11T20:55:13,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/fd1b2c4122f14189becd213a706785e1 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/fd1b2c4122f14189becd213a706785e1 2024-11-11T20:55:13,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/5ca2489638e8430db654399ac2ba5cbd to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/5ca2489638e8430db654399ac2ba5cbd 2024-11-11T20:55:13,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/96f3f04adce144d5abc7cb932144fa0b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/96f3f04adce144d5abc7cb932144fa0b 2024-11-11T20:55:13,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/49ce0c27fcd64f9eaa443ee41662b902 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/49ce0c27fcd64f9eaa443ee41662b902 2024-11-11T20:55:13,756 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/recovered.edits/445.seqid, newMaxSeqId=445, maxSeqId=1 2024-11-11T20:55:13,757 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. 2024-11-11T20:55:13,757 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] regionserver.HRegion(1676): Region close journal for 82ee51cab8163979e03e6fa77fa08aaf: Waiting for close lock at 1731358512371Running coprocessor pre-close hooks at 1731358512371Disabling compacts and flushes for region at 1731358512371Disabling writes for close at 1731358512372 (+1 ms)Obtaining lock to block concurrent updates at 1731358512372Preparing flush snapshotting stores in 82ee51cab8163979e03e6fa77fa08aaf at 1731358512372Finished memstore snapshotting TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731358512373 (+1 ms)Flushing stores of TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf. at 1731358512374 (+1 ms)Flushing 82ee51cab8163979e03e6fa77fa08aaf/A: creating writer at 1731358512374Flushing 82ee51cab8163979e03e6fa77fa08aaf/A: appending metadata at 1731358512380 (+6 ms)Flushing 82ee51cab8163979e03e6fa77fa08aaf/A: closing flushed file at 1731358512380Flushing 82ee51cab8163979e03e6fa77fa08aaf/B: creating writer at 1731358512794 (+414 ms)Flushing 82ee51cab8163979e03e6fa77fa08aaf/B: appending metadata at 1731358512797 (+3 ms)Flushing 82ee51cab8163979e03e6fa77fa08aaf/B: closing flushed file at 1731358512797Flushing 82ee51cab8163979e03e6fa77fa08aaf/C: creating writer at 1731358513210 (+413 ms)Flushing 82ee51cab8163979e03e6fa77fa08aaf/C: appending metadata at 1731358513214 (+4 ms)Flushing 82ee51cab8163979e03e6fa77fa08aaf/C: closing flushed file at 1731358513214Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@316edcde: reopening flushed file at 1731358513629 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41892943: reopening flushed file at 1731358513637 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@314cb84a: reopening flushed file at 1731358513643 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 82ee51cab8163979e03e6fa77fa08aaf in 1277ms, sequenceid=442, compaction requested=true at 1731358513649 (+6 ms)Writing region close event to WAL at 1731358513752 (+103 ms)Running coprocessor post-close hooks at 1731358513756 (+4 ms)Closed at 1731358513756 2024-11-11T20:55:13,759 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=126}] handler.UnassignRegionHandler(157): Closed 82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:55:13,760 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=125 updating hbase:meta row=82ee51cab8163979e03e6fa77fa08aaf, regionState=CLOSED 2024-11-11T20:55:13,762 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=126, ppid=125, state=RUNNABLE, hasLock=false; CloseRegionProcedure 82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 because future has completed 2024-11-11T20:55:13,764 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-11T20:55:13,765 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; CloseRegionProcedure 82ee51cab8163979e03e6fa77fa08aaf, server=51ca66f7ee3c,44669,1731358362066 in 1.5440 sec 2024-11-11T20:55:13,766 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=125, resume processing ppid=124 2024-11-11T20:55:13,767 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, ppid=124, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=82ee51cab8163979e03e6fa77fa08aaf, UNASSIGN in 1.5500 sec 2024-11-11T20:55:13,769 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-11T20:55:13,769 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5550 sec 2024-11-11T20:55:13,771 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358513770"}]},"ts":"1731358513770"} 2024-11-11T20:55:13,772 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T20:55:13,772 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T20:55:13,775 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5680 sec 2024-11-11T20:55:14,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-11T20:55:14,357 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:55:14,359 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T20:55:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:55:14,363 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=127, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:55:14,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T20:55:14,365 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=127, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:55:14,368 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:55:14,373 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A, FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B, FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C, FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/recovered.edits] 2024-11-11T20:55:14,379 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/18c740f82ff5418b8884a78664553c00 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/18c740f82ff5418b8884a78664553c00 2024-11-11T20:55:14,381 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/1fe192569a7944879d82c4102ea48247 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/1fe192569a7944879d82c4102ea48247 2024-11-11T20:55:14,383 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/5db9b63fecc74fd899e85392ae1679b9 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/5db9b63fecc74fd899e85392ae1679b9 2024-11-11T20:55:14,386 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/cb155b95e34a476e87f1b6460fc888f5 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/A/cb155b95e34a476e87f1b6460fc888f5 2024-11-11T20:55:14,389 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/0f7f0ff2afcb4c93b1193cd930d7d72a to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/0f7f0ff2afcb4c93b1193cd930d7d72a 2024-11-11T20:55:14,391 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/3638b9259e364792b25513b3e3d3343c to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/3638b9259e364792b25513b3e3d3343c 2024-11-11T20:55:14,393 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/6a3305c9d66b452fa3f03b7d390d752b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/6a3305c9d66b452fa3f03b7d390d752b 2024-11-11T20:55:14,395 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/89bd8ccb63aa444a90cb013b3012f8d5 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/B/89bd8ccb63aa444a90cb013b3012f8d5 2024-11-11T20:55:14,398 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/735e9be7e02a4f709abdab39f641a5da to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/735e9be7e02a4f709abdab39f641a5da 2024-11-11T20:55:14,400 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/ac2a55b0d71a49938e88de6f2730a14f to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/ac2a55b0d71a49938e88de6f2730a14f 2024-11-11T20:55:14,402 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/bec7ccb923824691839aeaf8ead15a31 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/bec7ccb923824691839aeaf8ead15a31 2024-11-11T20:55:14,404 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/f59274fca5604bf18e73cd866efa648d to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/C/f59274fca5604bf18e73cd866efa648d 2024-11-11T20:55:14,407 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/recovered.edits/445.seqid to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf/recovered.edits/445.seqid 2024-11-11T20:55:14,407 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/82ee51cab8163979e03e6fa77fa08aaf 2024-11-11T20:55:14,408 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T20:55:14,410 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=127, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:55:14,412 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T20:55:14,414 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T20:55:14,415 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=127, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:55:14,416 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T20:55:14,416 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731358514416"}]},"ts":"9223372036854775807"} 2024-11-11T20:55:14,418 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T20:55:14,418 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 82ee51cab8163979e03e6fa77fa08aaf, NAME => 'TestAcidGuarantees,,1731358486264.82ee51cab8163979e03e6fa77fa08aaf.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T20:55:14,418 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T20:55:14,418 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731358514418"}]},"ts":"9223372036854775807"} 2024-11-11T20:55:14,419 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T20:55:14,420 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=127, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:55:14,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 61 msec 2024-11-11T20:55:14,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T20:55:14,476 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T20:55:14,476 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:55:14,487 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=249 (was 252), OpenFileDescriptor=451 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=286 (was 308), ProcessCount=11 (was 11), AvailableMemoryMB=3999 (was 4028) 2024-11-11T20:55:14,499 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=249, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=286, ProcessCount=11, AvailableMemoryMB=3998 2024-11-11T20:55:14,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T20:55:14,501 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T20:55:14,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=128, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T20:55:14,503 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T20:55:14,504 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:55:14,504 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 128 2024-11-11T20:55:14,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-11T20:55:14,505 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T20:55:14,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742293_1469 (size=1061) 2024-11-11T20:55:14,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-11T20:55:14,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-11T20:55:14,916 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23 2024-11-11T20:55:14,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742294_1470 (size=53) 2024-11-11T20:55:15,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-11T20:55:15,330 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:55:15,330 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing be56906506ad5c3978d05dd159fda63b, disabling compactions & flushes 2024-11-11T20:55:15,330 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:15,330 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:15,331 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. after waiting 0 ms 2024-11-11T20:55:15,331 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:15,331 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:15,331 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for be56906506ad5c3978d05dd159fda63b: Waiting for close lock at 1731358515330Disabling compacts and flushes for region at 1731358515330Disabling writes for close at 1731358515331 (+1 ms)Writing region close event to WAL at 1731358515331Closed at 1731358515331 2024-11-11T20:55:15,334 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T20:55:15,335 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731358515334"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731358515334"}]},"ts":"1731358515334"} 2024-11-11T20:55:15,337 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T20:55:15,338 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T20:55:15,338 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358515338"}]},"ts":"1731358515338"} 2024-11-11T20:55:15,340 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T20:55:15,341 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=be56906506ad5c3978d05dd159fda63b, ASSIGN}] 2024-11-11T20:55:15,342 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=be56906506ad5c3978d05dd159fda63b, ASSIGN 2024-11-11T20:55:15,343 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=be56906506ad5c3978d05dd159fda63b, ASSIGN; state=OFFLINE, location=51ca66f7ee3c,44669,1731358362066; forceNewPlan=false, retain=false 2024-11-11T20:55:15,494 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=be56906506ad5c3978d05dd159fda63b, regionState=OPENING, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:15,496 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=129, ppid=128, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=be56906506ad5c3978d05dd159fda63b, ASSIGN because future has completed 2024-11-11T20:55:15,497 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; OpenRegionProcedure be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066}] 2024-11-11T20:55:15,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-11T20:55:15,656 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:15,657 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7752): Opening region: {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:55:15,658 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:15,658 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:55:15,658 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7794): checking encryption for be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:15,659 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(7797): checking classloading for be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:15,661 INFO [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:15,662 INFO [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T20:55:15,662 INFO [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be56906506ad5c3978d05dd159fda63b columnFamilyName A 2024-11-11T20:55:15,662 DEBUG [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:55:15,663 INFO [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] regionserver.HStore(327): Store=be56906506ad5c3978d05dd159fda63b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:55:15,663 INFO [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:15,664 INFO [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T20:55:15,664 INFO [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be56906506ad5c3978d05dd159fda63b columnFamilyName B 2024-11-11T20:55:15,664 DEBUG [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:55:15,665 INFO [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] regionserver.HStore(327): Store=be56906506ad5c3978d05dd159fda63b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:55:15,665 INFO [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:15,666 INFO [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T20:55:15,666 INFO [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be56906506ad5c3978d05dd159fda63b columnFamilyName C 2024-11-11T20:55:15,666 DEBUG [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:55:15,666 INFO [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] regionserver.HStore(327): Store=be56906506ad5c3978d05dd159fda63b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:55:15,667 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:15,667 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1038): replaying wal for be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:15,667 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:15,668 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:15,669 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1048): stopping wal replay for be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:15,669 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1060): Cleaning up temporary data for be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:15,669 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T20:55:15,671 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1093): writing seq id for be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:15,672 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T20:55:15,673 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1114): Opened be56906506ad5c3978d05dd159fda63b; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74831287, jitterRate=0.1150730699300766}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T20:55:15,673 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1122): Running coprocessor post-open hooks for be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:15,674 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegion(1006): Region open journal for be56906506ad5c3978d05dd159fda63b: Running coprocessor pre-open hook at 1731358515659Writing region info on filesystem at 1731358515659Initializing all the Stores at 1731358515660 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358515660Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358515660Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358515660Cleaning up temporary data from old regions at 1731358515669 (+9 ms)Running coprocessor post-open hooks at 1731358515673 (+4 ms)Region opened successfully at 1731358515673 2024-11-11T20:55:15,674 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., pid=130, masterSystemTime=1731358515650 2024-11-11T20:55:15,676 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:15,676 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=130}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:15,677 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=129 updating hbase:meta row=be56906506ad5c3978d05dd159fda63b, regionState=OPEN, openSeqNum=2, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:15,678 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=130, ppid=129, state=RUNNABLE, hasLock=false; OpenRegionProcedure be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 because future has completed 2024-11-11T20:55:15,681 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-11T20:55:15,681 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; OpenRegionProcedure be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 in 183 msec 2024-11-11T20:55:15,684 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-11-11T20:55:15,684 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=be56906506ad5c3978d05dd159fda63b, ASSIGN in 340 msec 2024-11-11T20:55:15,685 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T20:55:15,685 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358515685"}]},"ts":"1731358515685"} 2024-11-11T20:55:15,686 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T20:55:15,687 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=128, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T20:55:15,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1860 sec 2024-11-11T20:55:16,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-11-11T20:55:16,657 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:55:16,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5581e3e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:16,659 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:55:16,659 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:55:16,662 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:55:16,662 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:55:16,662 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:55:16,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c812665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:16,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:55:16,663 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:55:16,664 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:16,665 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38910, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:55:16,666 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@817813b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:16,666 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:55:16,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:55:16,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:55:16,669 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48692, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:55:16,670 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T20:55:16,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 51ca66f7ee3c,38429,1731358361462 2024-11-11T20:55:16,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2e053a0f 2024-11-11T20:55:16,670 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T20:55:16,671 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T20:55:16,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T20:55:16,673 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T20:55:16,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T20:55:16,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742295_1471 (size=1097) 2024-11-11T20:55:17,088 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-11T20:55:17,088 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-11T20:55:17,093 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T20:55:17,096 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=be56906506ad5c3978d05dd159fda63b, REOPEN/MOVE}] 2024-11-11T20:55:17,097 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=be56906506ad5c3978d05dd159fda63b, REOPEN/MOVE 2024-11-11T20:55:17,098 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=be56906506ad5c3978d05dd159fda63b, regionState=CLOSING, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:17,100 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=be56906506ad5c3978d05dd159fda63b, REOPEN/MOVE because future has completed 2024-11-11T20:55:17,101 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T20:55:17,101 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066}] 2024-11-11T20:55:17,254 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(122): Close be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:17,254 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T20:55:17,254 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1722): Closing be56906506ad5c3978d05dd159fda63b, disabling compactions & flushes 2024-11-11T20:55:17,254 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:17,254 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:17,254 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. after waiting 0 ms 2024-11-11T20:55:17,254 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:17,264 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T20:55:17,264 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:17,265 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1676): Region close journal for be56906506ad5c3978d05dd159fda63b: Waiting for close lock at 1731358517254Running coprocessor pre-close hooks at 1731358517254Disabling compacts and flushes for region at 1731358517254Disabling writes for close at 1731358517254Writing region close event to WAL at 1731358517255 (+1 ms)Running coprocessor post-close hooks at 1731358517264 (+9 ms)Closed at 1731358517264 2024-11-11T20:55:17,265 WARN [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionServer(3299): Not adding moved region record: be56906506ad5c3978d05dd159fda63b to self. 2024-11-11T20:55:17,266 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(157): Closed be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:17,267 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=be56906506ad5c3978d05dd159fda63b, regionState=CLOSED 2024-11-11T20:55:17,268 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 because future has completed 2024-11-11T20:55:17,272 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-11T20:55:17,272 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseRegionProcedure be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 in 168 msec 2024-11-11T20:55:17,272 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=be56906506ad5c3978d05dd159fda63b, REOPEN/MOVE; state=CLOSED, location=51ca66f7ee3c,44669,1731358362066; forceNewPlan=false, retain=true 2024-11-11T20:55:17,423 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=be56906506ad5c3978d05dd159fda63b, regionState=OPENING, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:17,426 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=be56906506ad5c3978d05dd159fda63b, REOPEN/MOVE because future has completed 2024-11-11T20:55:17,426 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=135, ppid=133, state=RUNNABLE, hasLock=false; OpenRegionProcedure be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066}] 2024-11-11T20:55:17,586 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:17,586 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegion(7752): Opening region: {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} 2024-11-11T20:55:17,587 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:17,587 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T20:55:17,588 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegion(7794): checking encryption for be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:17,588 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegion(7797): checking classloading for be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:17,590 INFO [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:17,591 INFO [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T20:55:17,591 INFO [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be56906506ad5c3978d05dd159fda63b columnFamilyName A 2024-11-11T20:55:17,593 DEBUG [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:55:17,593 INFO [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] regionserver.HStore(327): Store=be56906506ad5c3978d05dd159fda63b/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:55:17,594 INFO [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:17,594 INFO [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T20:55:17,594 INFO [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be56906506ad5c3978d05dd159fda63b columnFamilyName B 2024-11-11T20:55:17,594 DEBUG [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:55:17,595 INFO [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] regionserver.HStore(327): Store=be56906506ad5c3978d05dd159fda63b/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:55:17,595 INFO [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:17,595 INFO [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T20:55:17,596 INFO [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region be56906506ad5c3978d05dd159fda63b columnFamilyName C 2024-11-11T20:55:17,596 DEBUG [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:55:17,596 INFO [StoreOpener-be56906506ad5c3978d05dd159fda63b-1 {}] regionserver.HStore(327): Store=be56906506ad5c3978d05dd159fda63b/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T20:55:17,596 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:17,596 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegion(1038): replaying wal for be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:17,597 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:17,598 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:17,599 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegion(1048): stopping wal replay for be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:17,599 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegion(1060): Cleaning up temporary data for be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:17,599 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T20:55:17,600 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegion(1093): writing seq id for be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:17,601 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegion(1114): Opened be56906506ad5c3978d05dd159fda63b; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68252801, jitterRate=0.017045989632606506}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T20:55:17,601 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegion(1122): Running coprocessor post-open hooks for be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:17,602 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegion(1006): Region open journal for be56906506ad5c3978d05dd159fda63b: Running coprocessor pre-open hook at 1731358517588Writing region info on filesystem at 1731358517588Initializing all the Stores at 1731358517590 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358517590Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358517590Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731358517590Cleaning up temporary data from old regions at 1731358517599 (+9 ms)Running coprocessor post-open hooks at 1731358517601 (+2 ms)Region opened successfully at 1731358517602 (+1 ms) 2024-11-11T20:55:17,603 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., pid=135, masterSystemTime=1731358517579 2024-11-11T20:55:17,604 DEBUG [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:17,604 INFO [RS_OPEN_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_OPEN_REGION, pid=135}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:17,605 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=be56906506ad5c3978d05dd159fda63b, regionState=OPEN, openSeqNum=5, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:17,606 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=135, ppid=133, state=RUNNABLE, hasLock=false; OpenRegionProcedure be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 because future has completed 2024-11-11T20:55:17,608 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=135, resume processing ppid=133 2024-11-11T20:55:17,608 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, ppid=133, state=SUCCESS, hasLock=false; OpenRegionProcedure be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 in 180 msec 2024-11-11T20:55:17,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-11T20:55:17,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=be56906506ad5c3978d05dd159fda63b, REOPEN/MOVE in 512 msec 2024-11-11T20:55:17,611 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-11T20:55:17,611 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 517 msec 2024-11-11T20:55:17,613 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 939 msec 2024-11-11T20:55:17,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T20:55:17,613 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:55:17,614 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1572f078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,614 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:55:17,614 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:55:17,615 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:55:17,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:55:17,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:55:17,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24c34d79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:55:17,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:55:17,615 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:17,616 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:55:17,617 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64374a69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,617 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23448b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,617 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:55:17,618 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:55:17,618 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:55:17,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:55:17,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:55:17,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@311521cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:55:17,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:55:17,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:17,620 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:55:17,620 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b2dd1ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7db06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,621 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:55:17,621 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:55:17,622 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:55:17,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:55:17,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:55:17,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b531c5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:55:17,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:55:17,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:17,623 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38958, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:55:17,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e5e7d5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,624 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@763ca8cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,624 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:55:17,624 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:55:17,625 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:55:17,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:55:17,625 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:55:17,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ceb414b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:55:17,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:55:17,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:17,627 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38982, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:55:17,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73255333, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,628 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7122db2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,628 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:55:17,628 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:55:17,629 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:55:17,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:55:17,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:55:17,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ce9d884, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:55:17,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:55:17,630 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:17,631 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38988, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:55:17,631 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16410d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,632 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,632 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:55:17,632 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:55:17,633 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:55:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:55:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:55:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fd96b79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:55:17,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:55:17,634 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:17,634 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:55:17,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59307cc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,636 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb8a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,636 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:55:17,636 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:55:17,637 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:55:17,637 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:55:17,637 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:55:17,637 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b5df76c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,637 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:55:17,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:55:17,638 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:17,639 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39038, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:55:17,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d9b006, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,640 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@267cfa21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,640 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:55:17,640 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:55:17,641 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:55:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:55:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:55:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65018ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:55:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:55:17,641 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:17,642 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39054, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:55:17,642 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bab0920, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,643 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c58aaf1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,643 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:55:17,643 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:55:17,643 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:55:17,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:55:17,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:55:17,643 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@499f04f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:55:17,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:55:17,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:17,644 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39062, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:55:17,645 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a7e367, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,645 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d9bca83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,645 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 51ca66f7ee3c,38429,-1 for getting cluster id 2024-11-11T20:55:17,645 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T20:55:17,646 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8' 2024-11-11T20:55:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T20:55:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "e0a95a2a-d7fb-4c62-8ce6-8f4201ecb5b8" 2024-11-11T20:55:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e88959c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [51ca66f7ee3c,38429,-1] 2024-11-11T20:55:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T20:55:17,646 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:17,647 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39070, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T20:55:17,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1882e483, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T20:55:17,648 DEBUG [Thread-2142 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:55:17,648 DEBUG [Thread-2146 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:55:17,648 DEBUG [Thread-2135 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:55:17,648 DEBUG [Thread-2133 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:55:17,648 DEBUG [Thread-2150 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:55:17,648 DEBUG [Thread-2144 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:55:17,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:55:17,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:55:17,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:55:17,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:55:17,649 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:55:17,649 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:55:17,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:55:17,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:55:17,649 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:55:17,649 DEBUG [Thread-2150 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:55:17,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:55:17,650 DEBUG [Thread-2150 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:55:17,650 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48744, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:55:17,650 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48756, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:55:17,650 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48734, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:55:17,651 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48710, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:55:17,651 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48708, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:55:17,651 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48772, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:55:17,651 DEBUG [Thread-2148 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:55:17,649 DEBUG [Thread-2131 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:55:17,649 DEBUG [Thread-2139 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:55:17,649 DEBUG [Thread-2137 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T20:55:17,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:55:17,652 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:55:17,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:55:17,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:55:17,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:55:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:55:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=51ca66f7ee3c,44669,1731358362066, seqNum=-1] 2024-11-11T20:55:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:55:17,653 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T20:55:17,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=136, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees 2024-11-11T20:55:17,654 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:55:17,654 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48798, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:55:17,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:55:17,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:55:17,654 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:55:17,654 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48786, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:55:17,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-11T20:55:17,655 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48802, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T20:55:17,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:55:17,655 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:55:17,655 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=136, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:55:17,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:55:17,656 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:55:17,656 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:55:17,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:55:17,657 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5] 2024-11-11T20:55:17,658 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=136, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:55:17,658 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:55:17,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:17,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be56906506ad5c3978d05dd159fda63b 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T20:55:17,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=A 2024-11-11T20:55:17,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:17,662 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=B 2024-11-11T20:55:17,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:17,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=C 2024-11-11T20:55:17,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:17,681 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:17,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358577678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:17,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:17,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:17,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:17,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:17,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358577679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:17,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:17,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:17,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:17,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:17,685 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:17,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358577680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:17,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48786 deadline: 1731358577680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:17,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:17,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:17,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:17,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:17,685 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:17,685 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:17,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731358577681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:17,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b01ca1ba1fb2461dae657c884448e2b9_be56906506ad5c3978d05dd159fda63b is 50, key is test_row_0/A:col10/1731358517659/Put/seqid=0 2024-11-11T20:55:17,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742296_1472 (size=14594) 2024-11-11T20:55:17,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-11T20:55:17,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:17,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358577786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:17,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:17,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:17,789 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:17,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:17,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358577786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:17,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:17,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:17,789 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:17,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:17,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358577795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:17,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:17,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48786 deadline: 1731358577795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:17,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:17,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731358577795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:17,798 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:17,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-11T20:55:17,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:17,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:17,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:17,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:17,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:17,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:17,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-11T20:55:17,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:17,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:17,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:17,964 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:17,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:17,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:17,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-11T20:55:17,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:17,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358577996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:17,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:17,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:17,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358577996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:17,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:17,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:17,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:17,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:17,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:18,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:18,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358578005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:18,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:18,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:18,006 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:18,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731358578005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:18,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:18,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:18,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48786 deadline: 1731358578005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:18,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:18,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:18,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:18,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:18,008 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:18,105 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:55:18,108 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b01ca1ba1fb2461dae657c884448e2b9_be56906506ad5c3978d05dd159fda63b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b01ca1ba1fb2461dae657c884448e2b9_be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:18,108 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/be0f3c0549a5467b8a96219e05f67df3, store: [table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:18,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/be0f3c0549a5467b8a96219e05f67df3 is 175, key is test_row_0/A:col10/1731358517659/Put/seqid=0 2024-11-11T20:55:18,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742297_1473 (size=39549) 2024-11-11T20:55:18,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-11T20:55:18,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:18,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:18,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:18,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:18,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:18,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:18,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-11T20:55:18,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:18,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:18,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:18,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:18,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:18,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:18,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-11T20:55:18,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:18,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358578306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:18,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:18,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358578306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:18,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:18,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:18,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:18,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:18,308 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:18,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358578315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:18,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:18,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:18,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731358578315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:18,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:18,317 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:18,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:18,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:18,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48786 deadline: 1731358578315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:18,318 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:18,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:18,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:18,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:18,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-11T20:55:18,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:18,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:18,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:18,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:18,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:18,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:18,512 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/be0f3c0549a5467b8a96219e05f67df3 2024-11-11T20:55:18,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/93a8dff664da432fbf3eaff99d61b713 is 50, key is test_row_0/B:col10/1731358517659/Put/seqid=0 2024-11-11T20:55:18,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742298_1474 (size=12001) 2024-11-11T20:55:18,534 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/93a8dff664da432fbf3eaff99d61b713 2024-11-11T20:55:18,552 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/1627b74bdf0f47bb9752764d4b9bdf75 is 50, key is test_row_0/C:col10/1731358517659/Put/seqid=0 2024-11-11T20:55:18,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742299_1475 (size=12001) 2024-11-11T20:55:18,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-11T20:55:18,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:18,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:18,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:18,574 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:18,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:18,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-11T20:55:18,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:18,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:18,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:18,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:18,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:18,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:18,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-11T20:55:18,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:18,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358578815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:18,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:18,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358578815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:18,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:18,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:18,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:18,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:18,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:18,818 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:18,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358578825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:18,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:18,827 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:18,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731358578826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:18,827 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:18,827 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:18,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:18,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:18,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:18,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:18,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48786 deadline: 1731358578826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:18,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:18,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:18,828 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:18,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-11T20:55:18,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:18,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:18,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:18,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] handler.RSProcedureHandler(58): pid=137 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:18,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=137 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:18,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=137 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:18,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/1627b74bdf0f47bb9752764d4b9bdf75 2024-11-11T20:55:18,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/be0f3c0549a5467b8a96219e05f67df3 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/be0f3c0549a5467b8a96219e05f67df3 2024-11-11T20:55:18,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/be0f3c0549a5467b8a96219e05f67df3, entries=200, sequenceid=16, filesize=38.6 K 2024-11-11T20:55:18,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/93a8dff664da432fbf3eaff99d61b713 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/93a8dff664da432fbf3eaff99d61b713 2024-11-11T20:55:18,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/93a8dff664da432fbf3eaff99d61b713, entries=150, sequenceid=16, filesize=11.7 K 2024-11-11T20:55:18,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/1627b74bdf0f47bb9752764d4b9bdf75 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/1627b74bdf0f47bb9752764d4b9bdf75 2024-11-11T20:55:18,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/1627b74bdf0f47bb9752764d4b9bdf75, entries=150, sequenceid=16, filesize=11.7 K 2024-11-11T20:55:18,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for be56906506ad5c3978d05dd159fda63b in 1308ms, sequenceid=16, compaction requested=false 2024-11-11T20:55:18,969 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T20:55:18,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be56906506ad5c3978d05dd159fda63b: 2024-11-11T20:55:19,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=137 2024-11-11T20:55:19,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:19,032 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2902): Flushing be56906506ad5c3978d05dd159fda63b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T20:55:19,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=A 2024-11-11T20:55:19,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:19,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=B 2024-11-11T20:55:19,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:19,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=C 2024-11-11T20:55:19,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:19,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111174fa18d388a749dea32ea2e4491d4e38_be56906506ad5c3978d05dd159fda63b is 50, key is test_row_0/A:col10/1731358517679/Put/seqid=0 2024-11-11T20:55:19,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742300_1476 (size=12154) 2024-11-11T20:55:19,278 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T20:55:19,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:55:19,445 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111174fa18d388a749dea32ea2e4491d4e38_be56906506ad5c3978d05dd159fda63b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111174fa18d388a749dea32ea2e4491d4e38_be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:19,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/f3cc5f25da694622ae5a591ab5817fdf, store: [table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:19,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/f3cc5f25da694622ae5a591ab5817fdf is 175, key is test_row_0/A:col10/1731358517679/Put/seqid=0 2024-11-11T20:55:19,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742301_1477 (size=30955) 2024-11-11T20:55:19,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-11T20:55:19,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:19,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:19,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:19,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48786 deadline: 1731358579835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:19,840 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in be56906506ad5c3978d05dd159fda63b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2161 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:55:19,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:19,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:19,841 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:19,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:19,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358579838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:19,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:19,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:19,842 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:19,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:19,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358579839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:19,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:19,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:19,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:19,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:19,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358579840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:19,844 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:19,844 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:19,844 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:19,847 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:19,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731358579845, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:19,848 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_0' in be56906506ad5c3978d05dd159fda63b of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:55:19,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:19,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:19,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:19,850 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/f3cc5f25da694622ae5a591ab5817fdf 2024-11-11T20:55:19,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/c33a331605d04ee8be1ada102029750a is 50, key is test_row_0/B:col10/1731358517679/Put/seqid=0 2024-11-11T20:55:19,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742302_1478 (size=12001) 2024-11-11T20:55:19,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:19,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358579945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:19,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:19,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:19,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:19,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:19,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358579956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:19,958 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:19,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:19,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:19,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:19,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358579956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:19,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:19,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:19,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:20,157 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:20,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358580155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:20,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:20,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:20,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:20,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:20,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358580165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:20,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:20,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:20,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:20,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:20,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358580165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:20,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:20,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:20,168 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:20,273 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/c33a331605d04ee8be1ada102029750a 2024-11-11T20:55:20,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/437af3df1ac14d6a86cc3b83f493b133 is 50, key is test_row_0/C:col10/1731358517679/Put/seqid=0 2024-11-11T20:55:20,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742303_1479 (size=12001) 2024-11-11T20:55:20,467 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:20,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358580466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:20,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:20,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:20,468 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:20,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:20,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358580475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:20,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:20,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:20,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:20,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358580475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:20,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:20,477 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:20,477 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:20,477 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:20,682 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/437af3df1ac14d6a86cc3b83f493b133 2024-11-11T20:55:20,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/f3cc5f25da694622ae5a591ab5817fdf as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/f3cc5f25da694622ae5a591ab5817fdf 2024-11-11T20:55:20,689 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/f3cc5f25da694622ae5a591ab5817fdf, entries=150, sequenceid=36, filesize=30.2 K 2024-11-11T20:55:20,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/c33a331605d04ee8be1ada102029750a as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/c33a331605d04ee8be1ada102029750a 2024-11-11T20:55:20,692 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/c33a331605d04ee8be1ada102029750a, entries=150, sequenceid=36, filesize=11.7 K 2024-11-11T20:55:20,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/437af3df1ac14d6a86cc3b83f493b133 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/437af3df1ac14d6a86cc3b83f493b133 2024-11-11T20:55:20,702 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/437af3df1ac14d6a86cc3b83f493b133, entries=150, sequenceid=36, filesize=11.7 K 2024-11-11T20:55:20,703 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for be56906506ad5c3978d05dd159fda63b in 1670ms, sequenceid=36, compaction requested=false 2024-11-11T20:55:20,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.HRegion(2603): Flush status journal for be56906506ad5c3978d05dd159fda63b: 2024-11-11T20:55:20,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:20,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=137}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=137 2024-11-11T20:55:20,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=137 2024-11-11T20:55:20,705 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-11T20:55:20,705 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0460 sec 2024-11-11T20:55:20,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=136, table=TestAcidGuarantees in 3.0540 sec 2024-11-11T20:55:20,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:20,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be56906506ad5c3978d05dd159fda63b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T20:55:20,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=A 2024-11-11T20:55:20,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:20,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=B 2024-11-11T20:55:20,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:20,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=C 2024-11-11T20:55:20,978 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:20,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112a161b804c3f419dbe9571bae9c80f36_be56906506ad5c3978d05dd159fda63b is 50, key is test_row_0/A:col10/1731358519838/Put/seqid=0 2024-11-11T20:55:20,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742304_1480 (size=14594) 2024-11-11T20:55:21,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:21,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358581003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:21,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:21,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:21,007 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:21,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358581006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:21,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:21,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:21,011 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:21,011 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:21,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358581006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:21,011 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:21,011 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:21,011 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:21,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:21,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358581115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:21,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:21,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358581115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:21,118 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:21,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:21,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358581115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:21,119 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:21,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:21,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358581326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:21,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:21,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358581326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:21,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:21,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:21,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:21,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:21,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358581326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:21,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:21,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:21,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:21,389 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:55:21,392 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112a161b804c3f419dbe9571bae9c80f36_be56906506ad5c3978d05dd159fda63b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112a161b804c3f419dbe9571bae9c80f36_be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:21,393 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/4daae7d3fb02488ea432ac24947275ab, store: [table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:21,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/4daae7d3fb02488ea432ac24947275ab is 175, key is test_row_0/A:col10/1731358519838/Put/seqid=0 2024-11-11T20:55:21,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742305_1481 (size=39549) 2024-11-11T20:55:21,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:21,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358581635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:21,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:21,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:21,639 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:21,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358581636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:21,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:21,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:21,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:21,639 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:21,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358581636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:21,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:21,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:21,639 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:21,796 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/4daae7d3fb02488ea432ac24947275ab 2024-11-11T20:55:21,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/cdf97d63202b47909409a5df0fa072db is 50, key is test_row_0/B:col10/1731358519838/Put/seqid=0 2024-11-11T20:55:21,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742306_1482 (size=12001) 2024-11-11T20:55:21,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-11T20:55:21,836 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:55:21,836 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:55:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees 2024-11-11T20:55:21,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-11T20:55:21,837 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:55:21,838 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=138, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:55:21,838 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:55:21,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:21,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48786 deadline: 1731358581866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:21,868 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in be56906506ad5c3978d05dd159fda63b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:55:21,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:21,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:21,869 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:21,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:21,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731358581876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:21,878 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_0' in be56906506ad5c3978d05dd159fda63b of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:55:21,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:21,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:21,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:21,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-11T20:55:21,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-11T20:55:21,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:21,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:21,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:21,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:21,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:21,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:22,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-11T20:55:22,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:22,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:22,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:22,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:22,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:22,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:22,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358582146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:22,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:22,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:22,148 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:22,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:22,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358582146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:22,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:22,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:22,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:22,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:22,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358582146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:22,149 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:22,149 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:22,149 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-11T20:55:22,209 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/cdf97d63202b47909409a5df0fa072db 2024-11-11T20:55:22,215 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/4049bd6d5945447592a28376fcea8184 is 50, key is test_row_0/C:col10/1731358519838/Put/seqid=0 2024-11-11T20:55:22,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742307_1483 (size=12001) 2024-11-11T20:55:22,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-11T20:55:22,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:22,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:22,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:22,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:22,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:22,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:22,448 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-11T20:55:22,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:22,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:22,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:22,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:22,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:22,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-11T20:55:22,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-11T20:55:22,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:22,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:22,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:22,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] handler.RSProcedureHandler(58): pid=139 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:22,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=139 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:22,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=139 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:22,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/4049bd6d5945447592a28376fcea8184 2024-11-11T20:55:22,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/4daae7d3fb02488ea432ac24947275ab as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/4daae7d3fb02488ea432ac24947275ab 2024-11-11T20:55:22,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/4daae7d3fb02488ea432ac24947275ab, entries=200, sequenceid=49, filesize=38.6 K 2024-11-11T20:55:22,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/cdf97d63202b47909409a5df0fa072db as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/cdf97d63202b47909409a5df0fa072db 2024-11-11T20:55:22,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/cdf97d63202b47909409a5df0fa072db, entries=150, sequenceid=49, filesize=11.7 K 2024-11-11T20:55:22,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/4049bd6d5945447592a28376fcea8184 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/4049bd6d5945447592a28376fcea8184 2024-11-11T20:55:22,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/4049bd6d5945447592a28376fcea8184, entries=150, sequenceid=49, filesize=11.7 K 2024-11-11T20:55:22,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for be56906506ad5c3978d05dd159fda63b in 1653ms, sequenceid=49, compaction requested=true 2024-11-11T20:55:22,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be56906506ad5c3978d05dd159fda63b: 2024-11-11T20:55:22,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be56906506ad5c3978d05dd159fda63b:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:55:22,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:22,631 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:55:22,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be56906506ad5c3978d05dd159fda63b:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:55:22,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:22,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be56906506ad5c3978d05dd159fda63b:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:55:22,631 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:55:22,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:55:22,631 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:55:22,631 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:55:22,631 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): be56906506ad5c3978d05dd159fda63b/B is initiating minor compaction (all files) 2024-11-11T20:55:22,631 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): be56906506ad5c3978d05dd159fda63b/A is initiating minor compaction (all files) 2024-11-11T20:55:22,631 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be56906506ad5c3978d05dd159fda63b/A in TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:22,631 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be56906506ad5c3978d05dd159fda63b/B in TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:22,631 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/be0f3c0549a5467b8a96219e05f67df3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/f3cc5f25da694622ae5a591ab5817fdf, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/4daae7d3fb02488ea432ac24947275ab] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp, totalSize=107.5 K 2024-11-11T20:55:22,631 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/93a8dff664da432fbf3eaff99d61b713, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/c33a331605d04ee8be1ada102029750a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/cdf97d63202b47909409a5df0fa072db] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp, totalSize=35.2 K 2024-11-11T20:55:22,631 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:22,632 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. files: [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/be0f3c0549a5467b8a96219e05f67df3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/f3cc5f25da694622ae5a591ab5817fdf, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/4daae7d3fb02488ea432ac24947275ab] 2024-11-11T20:55:22,632 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting be0f3c0549a5467b8a96219e05f67df3, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731358517658 2024-11-11T20:55:22,632 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 93a8dff664da432fbf3eaff99d61b713, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731358517658 2024-11-11T20:55:22,632 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting c33a331605d04ee8be1ada102029750a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731358517668 2024-11-11T20:55:22,632 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting f3cc5f25da694622ae5a591ab5817fdf, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731358517668 2024-11-11T20:55:22,632 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting cdf97d63202b47909409a5df0fa072db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731358519831 2024-11-11T20:55:22,632 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4daae7d3fb02488ea432ac24947275ab, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731358519831 2024-11-11T20:55:22,636 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:22,638 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be56906506ad5c3978d05dd159fda63b#B#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:55:22,638 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/427230f421a04a289e333694e330f59a is 50, key is test_row_0/B:col10/1731358519838/Put/seqid=0 2024-11-11T20:55:22,639 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411118c10bf3670b9427e9f11215fe7b37457_be56906506ad5c3978d05dd159fda63b store=[table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:22,642 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411118c10bf3670b9427e9f11215fe7b37457_be56906506ad5c3978d05dd159fda63b, store=[table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:22,642 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118c10bf3670b9427e9f11215fe7b37457_be56906506ad5c3978d05dd159fda63b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:22,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742308_1484 (size=12104) 2024-11-11T20:55:22,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742309_1485 (size=4469) 2024-11-11T20:55:22,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=139 2024-11-11T20:55:22,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:22,753 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2902): Flushing be56906506ad5c3978d05dd159fda63b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T20:55:22,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=A 2024-11-11T20:55:22,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:22,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=B 2024-11-11T20:55:22,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:22,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=C 2024-11-11T20:55:22,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:22,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b98abb3615b0468d902d951a05d7bbd9_be56906506ad5c3978d05dd159fda63b is 50, key is test_row_0/A:col10/1731358521005/Put/seqid=0 2024-11-11T20:55:22,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742310_1486 (size=12154) 2024-11-11T20:55:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-11T20:55:23,047 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/427230f421a04a289e333694e330f59a as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/427230f421a04a289e333694e330f59a 2024-11-11T20:55:23,048 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be56906506ad5c3978d05dd159fda63b#A#compaction#409 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:55:23,048 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/5ffeface8cdc42b79063e9acc4baa192 is 175, key is test_row_0/A:col10/1731358519838/Put/seqid=0 2024-11-11T20:55:23,051 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be56906506ad5c3978d05dd159fda63b/B of be56906506ad5c3978d05dd159fda63b into 427230f421a04a289e333694e330f59a(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:55:23,051 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be56906506ad5c3978d05dd159fda63b: 2024-11-11T20:55:23,051 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., storeName=be56906506ad5c3978d05dd159fda63b/B, priority=13, startTime=1731358522631; duration=0sec 2024-11-11T20:55:23,051 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:55:23,051 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be56906506ad5c3978d05dd159fda63b:B 2024-11-11T20:55:23,051 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:55:23,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742311_1487 (size=31058) 2024-11-11T20:55:23,053 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:55:23,053 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): be56906506ad5c3978d05dd159fda63b/C is initiating minor compaction (all files) 2024-11-11T20:55:23,053 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be56906506ad5c3978d05dd159fda63b/C in TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:23,053 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/1627b74bdf0f47bb9752764d4b9bdf75, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/437af3df1ac14d6a86cc3b83f493b133, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/4049bd6d5945447592a28376fcea8184] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp, totalSize=35.2 K 2024-11-11T20:55:23,053 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 1627b74bdf0f47bb9752764d4b9bdf75, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731358517658 2024-11-11T20:55:23,054 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 437af3df1ac14d6a86cc3b83f493b133, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731358517668 2024-11-11T20:55:23,054 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 4049bd6d5945447592a28376fcea8184, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731358519831 2024-11-11T20:55:23,056 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/5ffeface8cdc42b79063e9acc4baa192 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/5ffeface8cdc42b79063e9acc4baa192 2024-11-11T20:55:23,060 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be56906506ad5c3978d05dd159fda63b#C#compaction#411 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:55:23,061 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/b2e1282b8c4946d3bb6628e2fa505ad1 is 50, key is test_row_0/C:col10/1731358519838/Put/seqid=0 2024-11-11T20:55:23,063 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be56906506ad5c3978d05dd159fda63b/A of be56906506ad5c3978d05dd159fda63b into 5ffeface8cdc42b79063e9acc4baa192(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:55:23,063 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be56906506ad5c3978d05dd159fda63b: 2024-11-11T20:55:23,063 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., storeName=be56906506ad5c3978d05dd159fda63b/A, priority=13, startTime=1731358522630; duration=0sec 2024-11-11T20:55:23,063 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:23,063 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be56906506ad5c3978d05dd159fda63b:A 2024-11-11T20:55:23,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742312_1488 (size=12104) 2024-11-11T20:55:23,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:23,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:23,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:55:23,165 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b98abb3615b0468d902d951a05d7bbd9_be56906506ad5c3978d05dd159fda63b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b98abb3615b0468d902d951a05d7bbd9_be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:23,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/aa8705187b844618a676e226d3024dd0, store: [table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:23,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/aa8705187b844618a676e226d3024dd0 is 175, key is test_row_0/A:col10/1731358521005/Put/seqid=0 2024-11-11T20:55:23,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742313_1489 (size=30955) 2024-11-11T20:55:23,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:23,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358583171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:23,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:23,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:23,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:23,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:23,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358583172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:23,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:23,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:23,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358583173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:23,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:23,177 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:23,177 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:23,177 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:23,177 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:23,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358583285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:23,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:23,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358583286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:23,288 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:23,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:23,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358583286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:23,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:23,468 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/b2e1282b8c4946d3bb6628e2fa505ad1 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/b2e1282b8c4946d3bb6628e2fa505ad1 2024-11-11T20:55:23,471 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be56906506ad5c3978d05dd159fda63b/C of be56906506ad5c3978d05dd159fda63b into b2e1282b8c4946d3bb6628e2fa505ad1(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:55:23,471 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be56906506ad5c3978d05dd159fda63b: 2024-11-11T20:55:23,471 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., storeName=be56906506ad5c3978d05dd159fda63b/C, priority=13, startTime=1731358522631; duration=0sec 2024-11-11T20:55:23,471 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:23,471 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be56906506ad5c3978d05dd159fda63b:C 2024-11-11T20:55:23,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358583495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:23,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358583495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:23,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358583495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:23,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:23,569 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=69, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/aa8705187b844618a676e226d3024dd0 2024-11-11T20:55:23,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/bfc10be5d45947d886f1ec252b57460b is 50, key is test_row_0/B:col10/1731358521005/Put/seqid=0 2024-11-11T20:55:23,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742314_1490 (size=12001) 2024-11-11T20:55:23,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:23,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358583805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:23,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:23,806 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:23,806 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:23,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358583806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:23,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:23,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:23,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:23,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:23,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358583806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:23,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:23,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:23,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:23,979 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/bfc10be5d45947d886f1ec252b57460b 2024-11-11T20:55:23,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/5e16603363664108a637e153d68b1cb6 is 50, key is test_row_0/C:col10/1731358521005/Put/seqid=0 2024-11-11T20:55:23,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-11T20:55:23,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742315_1491 (size=12001) 2024-11-11T20:55:23,988 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/5e16603363664108a637e153d68b1cb6 2024-11-11T20:55:23,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/aa8705187b844618a676e226d3024dd0 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/aa8705187b844618a676e226d3024dd0 2024-11-11T20:55:23,995 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/aa8705187b844618a676e226d3024dd0, entries=150, sequenceid=69, filesize=30.2 K 2024-11-11T20:55:23,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/bfc10be5d45947d886f1ec252b57460b as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/bfc10be5d45947d886f1ec252b57460b 2024-11-11T20:55:23,999 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/bfc10be5d45947d886f1ec252b57460b, entries=150, sequenceid=69, filesize=11.7 K 2024-11-11T20:55:24,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/5e16603363664108a637e153d68b1cb6 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/5e16603363664108a637e153d68b1cb6 2024-11-11T20:55:24,004 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/5e16603363664108a637e153d68b1cb6, entries=150, sequenceid=69, filesize=11.7 K 2024-11-11T20:55:24,004 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for be56906506ad5c3978d05dd159fda63b in 1251ms, sequenceid=69, compaction requested=false 2024-11-11T20:55:24,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.HRegion(2603): Flush status journal for be56906506ad5c3978d05dd159fda63b: 2024-11-11T20:55:24,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:24,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=139}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=139 2024-11-11T20:55:24,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=139 2024-11-11T20:55:24,008 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-11T20:55:24,008 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1680 sec 2024-11-11T20:55:24,009 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=138, table=TestAcidGuarantees in 2.1720 sec 2024-11-11T20:55:24,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:24,318 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be56906506ad5c3978d05dd159fda63b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T20:55:24,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=A 2024-11-11T20:55:24,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:24,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=B 2024-11-11T20:55:24,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:24,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=C 2024-11-11T20:55:24,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:24,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ccbbe8a4c55f4d10a4bfd74fe429c630_be56906506ad5c3978d05dd159fda63b is 50, key is test_row_0/A:col10/1731358523172/Put/seqid=0 2024-11-11T20:55:24,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742316_1492 (size=14594) 2024-11-11T20:55:24,346 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:24,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358584342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:24,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:24,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:24,346 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:24,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:24,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358584342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:24,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:24,348 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:24,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:24,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:24,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358584342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:24,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:24,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:24,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:24,458 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:24,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358584456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:24,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:24,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:24,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358584456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:24,459 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:24,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:24,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:24,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358584456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:24,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:24,459 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:24,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:24,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:24,459 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:24,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358584666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:24,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358584666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:24,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:24,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:24,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:24,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:24,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:24,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358584666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:24,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:24,668 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:24,669 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:24,729 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:55:24,731 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ccbbe8a4c55f4d10a4bfd74fe429c630_be56906506ad5c3978d05dd159fda63b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ccbbe8a4c55f4d10a4bfd74fe429c630_be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:24,732 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/4ae607f23dfb4ac9b79dea0f193e4e19, store: [table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:24,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/4ae607f23dfb4ac9b79dea0f193e4e19 is 175, key is test_row_0/A:col10/1731358523172/Put/seqid=0 2024-11-11T20:55:24,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742317_1493 (size=39549) 2024-11-11T20:55:24,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:24,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358584975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:24,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358584975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:24,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:24,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:24,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:24,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:24,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:24,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:24,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:24,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358584975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:24,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:24,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:24,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:25,136 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/4ae607f23dfb4ac9b79dea0f193e4e19 2024-11-11T20:55:25,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/2c24de5a8c364aa196aa51258b0523a3 is 50, key is test_row_0/B:col10/1731358523172/Put/seqid=0 2024-11-11T20:55:25,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742318_1494 (size=12001) 2024-11-11T20:55:25,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:25,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358585485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:25,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:25,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358585485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:25,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:25,488 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:25,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358585485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:25,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:25,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:25,489 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:25,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/2c24de5a8c364aa196aa51258b0523a3 2024-11-11T20:55:25,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/4fe91cc4e4f349fbbd468539325fee9d is 50, key is test_row_0/C:col10/1731358523172/Put/seqid=0 2024-11-11T20:55:25,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742319_1495 (size=12001) 2024-11-11T20:55:25,886 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:25,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48786 deadline: 1731358585886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:25,887 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in be56906506ad5c3978d05dd159fda63b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:55:25,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:25,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:25,887 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:25,889 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:25,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731358585886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:25,889 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_0' in be56906506ad5c3978d05dd159fda63b of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:55:25,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:25,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:25,890 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:25,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/4fe91cc4e4f349fbbd468539325fee9d 2024-11-11T20:55:25,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/4ae607f23dfb4ac9b79dea0f193e4e19 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/4ae607f23dfb4ac9b79dea0f193e4e19 2024-11-11T20:55:25,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/4ae607f23dfb4ac9b79dea0f193e4e19, entries=200, sequenceid=86, filesize=38.6 K 2024-11-11T20:55:25,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/2c24de5a8c364aa196aa51258b0523a3 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/2c24de5a8c364aa196aa51258b0523a3 2024-11-11T20:55:25,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/2c24de5a8c364aa196aa51258b0523a3, entries=150, sequenceid=86, filesize=11.7 K 2024-11-11T20:55:25,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/4fe91cc4e4f349fbbd468539325fee9d as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/4fe91cc4e4f349fbbd468539325fee9d 2024-11-11T20:55:25,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/4fe91cc4e4f349fbbd468539325fee9d, entries=150, sequenceid=86, filesize=11.7 K 2024-11-11T20:55:25,967 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for be56906506ad5c3978d05dd159fda63b in 1649ms, sequenceid=86, compaction requested=true 2024-11-11T20:55:25,967 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be56906506ad5c3978d05dd159fda63b: 2024-11-11T20:55:25,967 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:55:25,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be56906506ad5c3978d05dd159fda63b:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:55:25,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:25,968 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:55:25,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be56906506ad5c3978d05dd159fda63b:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:55:25,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:25,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be56906506ad5c3978d05dd159fda63b:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:55:25,968 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:55:25,968 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:55:25,968 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): be56906506ad5c3978d05dd159fda63b/A is initiating minor compaction (all files) 2024-11-11T20:55:25,968 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be56906506ad5c3978d05dd159fda63b/A in TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:25,968 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/5ffeface8cdc42b79063e9acc4baa192, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/aa8705187b844618a676e226d3024dd0, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/4ae607f23dfb4ac9b79dea0f193e4e19] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp, totalSize=99.2 K 2024-11-11T20:55:25,969 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:25,969 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. files: [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/5ffeface8cdc42b79063e9acc4baa192, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/aa8705187b844618a676e226d3024dd0, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/4ae607f23dfb4ac9b79dea0f193e4e19] 2024-11-11T20:55:25,969 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:55:25,969 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): be56906506ad5c3978d05dd159fda63b/B is initiating minor compaction (all files) 2024-11-11T20:55:25,969 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be56906506ad5c3978d05dd159fda63b/B in TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:25,969 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ffeface8cdc42b79063e9acc4baa192, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731358519831 2024-11-11T20:55:25,969 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/427230f421a04a289e333694e330f59a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/bfc10be5d45947d886f1ec252b57460b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/2c24de5a8c364aa196aa51258b0523a3] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp, totalSize=35.3 K 2024-11-11T20:55:25,969 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa8705187b844618a676e226d3024dd0, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731358521002 2024-11-11T20:55:25,969 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 427230f421a04a289e333694e330f59a, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731358519831 2024-11-11T20:55:25,969 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ae607f23dfb4ac9b79dea0f193e4e19, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731358523161 2024-11-11T20:55:25,969 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting bfc10be5d45947d886f1ec252b57460b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731358521002 2024-11-11T20:55:25,970 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c24de5a8c364aa196aa51258b0523a3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731358523172 2024-11-11T20:55:25,975 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:25,978 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be56906506ad5c3978d05dd159fda63b#B#compaction#418 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:55:25,978 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/54e9eb0337b4414cb5a6192394f8794d is 50, key is test_row_0/B:col10/1731358523172/Put/seqid=0 2024-11-11T20:55:25,989 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111af9d23e7594742ecb28f67aa44d92aec_be56906506ad5c3978d05dd159fda63b store=[table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:25,991 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111af9d23e7594742ecb28f67aa44d92aec_be56906506ad5c3978d05dd159fda63b, store=[table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:25,991 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111af9d23e7594742ecb28f67aa44d92aec_be56906506ad5c3978d05dd159fda63b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:25,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742320_1496 (size=12207) 2024-11-11T20:55:25,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742321_1497 (size=4469) 2024-11-11T20:55:25,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=138 2024-11-11T20:55:25,996 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:55:25,996 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:55:25,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees 2024-11-11T20:55:25,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-11T20:55:25,999 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:55:26,000 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=140, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:55:26,000 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:55:26,009 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/54e9eb0337b4414cb5a6192394f8794d as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/54e9eb0337b4414cb5a6192394f8794d 2024-11-11T20:55:26,014 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be56906506ad5c3978d05dd159fda63b/B of be56906506ad5c3978d05dd159fda63b into 54e9eb0337b4414cb5a6192394f8794d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:55:26,014 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be56906506ad5c3978d05dd159fda63b: 2024-11-11T20:55:26,014 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., storeName=be56906506ad5c3978d05dd159fda63b/B, priority=13, startTime=1731358525968; duration=0sec 2024-11-11T20:55:26,014 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:55:26,014 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be56906506ad5c3978d05dd159fda63b:B 2024-11-11T20:55:26,014 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:55:26,015 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:55:26,015 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): be56906506ad5c3978d05dd159fda63b/C is initiating minor compaction (all files) 2024-11-11T20:55:26,015 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be56906506ad5c3978d05dd159fda63b/C in TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:26,015 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/b2e1282b8c4946d3bb6628e2fa505ad1, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/5e16603363664108a637e153d68b1cb6, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/4fe91cc4e4f349fbbd468539325fee9d] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp, totalSize=35.3 K 2024-11-11T20:55:26,015 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting b2e1282b8c4946d3bb6628e2fa505ad1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731358519831 2024-11-11T20:55:26,015 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e16603363664108a637e153d68b1cb6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731358521002 2024-11-11T20:55:26,016 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fe91cc4e4f349fbbd468539325fee9d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731358523172 2024-11-11T20:55:26,022 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be56906506ad5c3978d05dd159fda63b#C#compaction#419 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:55:26,023 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/1fa941b936924ed58addf5524a26b5f5 is 50, key is test_row_0/C:col10/1731358523172/Put/seqid=0 2024-11-11T20:55:26,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742322_1498 (size=12207) 2024-11-11T20:55:26,032 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/1fa941b936924ed58addf5524a26b5f5 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/1fa941b936924ed58addf5524a26b5f5 2024-11-11T20:55:26,036 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be56906506ad5c3978d05dd159fda63b/C of be56906506ad5c3978d05dd159fda63b into 1fa941b936924ed58addf5524a26b5f5(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:55:26,036 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be56906506ad5c3978d05dd159fda63b: 2024-11-11T20:55:26,036 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., storeName=be56906506ad5c3978d05dd159fda63b/C, priority=13, startTime=1731358525968; duration=0sec 2024-11-11T20:55:26,036 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:26,037 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be56906506ad5c3978d05dd159fda63b:C 2024-11-11T20:55:26,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-11T20:55:26,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=141 2024-11-11T20:55:26,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:26,153 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2902): Flushing be56906506ad5c3978d05dd159fda63b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T20:55:26,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=A 2024-11-11T20:55:26,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:26,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=B 2024-11-11T20:55:26,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:26,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=C 2024-11-11T20:55:26,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:26,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d39d1a7d20634ac983b2350e250b6af8_be56906506ad5c3978d05dd159fda63b is 50, key is test_row_0/A:col10/1731358524341/Put/seqid=0 2024-11-11T20:55:26,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742323_1499 (size=12154) 2024-11-11T20:55:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-11T20:55:26,396 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be56906506ad5c3978d05dd159fda63b#A#compaction#417 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:55:26,396 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/05fdc3c97d6e46b8ac605e7001012451 is 175, key is test_row_0/A:col10/1731358523172/Put/seqid=0 2024-11-11T20:55:26,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742324_1500 (size=31161) 2024-11-11T20:55:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:26,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:26,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:26,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358586516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:26,520 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:26,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:26,521 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:26,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:26,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358586517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:26,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:26,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:26,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:26,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:26,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358586517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:26,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:26,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:26,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:26,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:55:26,567 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d39d1a7d20634ac983b2350e250b6af8_be56906506ad5c3978d05dd159fda63b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d39d1a7d20634ac983b2350e250b6af8_be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:26,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/d596b833e82541cca9b4d06cbe719152, store: [table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:26,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/d596b833e82541cca9b4d06cbe719152 is 175, key is test_row_0/A:col10/1731358524341/Put/seqid=0 2024-11-11T20:55:26,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742325_1501 (size=30955) 2024-11-11T20:55:26,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-11T20:55:26,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:26,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358586626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:26,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:26,628 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:26,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:26,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358586626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:26,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:26,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:26,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:26,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358586626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:26,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:26,629 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:26,629 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:26,629 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:26,808 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/05fdc3c97d6e46b8ac605e7001012451 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/05fdc3c97d6e46b8ac605e7001012451 2024-11-11T20:55:26,811 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be56906506ad5c3978d05dd159fda63b/A of be56906506ad5c3978d05dd159fda63b into 05fdc3c97d6e46b8ac605e7001012451(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:55:26,811 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be56906506ad5c3978d05dd159fda63b: 2024-11-11T20:55:26,811 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., storeName=be56906506ad5c3978d05dd159fda63b/A, priority=13, startTime=1731358525967; duration=0sec 2024-11-11T20:55:26,811 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:26,812 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be56906506ad5c3978d05dd159fda63b:A 2024-11-11T20:55:26,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:26,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358586836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:26,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:26,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:26,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:26,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:26,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358586836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:26,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:26,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:26,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:26,839 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:26,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358586836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:26,839 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:26,839 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:26,839 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:26,972 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=106, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/d596b833e82541cca9b4d06cbe719152 2024-11-11T20:55:26,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/bf971f5dd91645eca4045d02ed96a8d9 is 50, key is test_row_0/B:col10/1731358524341/Put/seqid=0 2024-11-11T20:55:26,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742326_1502 (size=12001) 2024-11-11T20:55:27,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-11T20:55:27,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358587145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:27,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:27,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:27,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:27,149 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:27,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358587145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:27,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:27,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:27,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:27,150 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:27,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358587146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:27,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:27,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:27,150 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:27,381 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/bf971f5dd91645eca4045d02ed96a8d9 2024-11-11T20:55:27,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/a9cab78c72af4540b644d29506c4ebe3 is 50, key is test_row_0/C:col10/1731358524341/Put/seqid=0 2024-11-11T20:55:27,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742327_1503 (size=12001) 2024-11-11T20:55:27,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:27,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358587655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:27,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:27,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:27,657 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358587655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:27,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:27,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358587655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:27,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:27,790 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/a9cab78c72af4540b644d29506c4ebe3 2024-11-11T20:55:27,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/d596b833e82541cca9b4d06cbe719152 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/d596b833e82541cca9b4d06cbe719152 2024-11-11T20:55:27,795 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/d596b833e82541cca9b4d06cbe719152, entries=150, sequenceid=106, filesize=30.2 K 2024-11-11T20:55:27,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/bf971f5dd91645eca4045d02ed96a8d9 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/bf971f5dd91645eca4045d02ed96a8d9 2024-11-11T20:55:27,799 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/bf971f5dd91645eca4045d02ed96a8d9, entries=150, sequenceid=106, filesize=11.7 K 2024-11-11T20:55:27,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/a9cab78c72af4540b644d29506c4ebe3 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/a9cab78c72af4540b644d29506c4ebe3 2024-11-11T20:55:27,802 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/a9cab78c72af4540b644d29506c4ebe3, entries=150, sequenceid=106, filesize=11.7 K 2024-11-11T20:55:27,803 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for be56906506ad5c3978d05dd159fda63b in 1650ms, sequenceid=106, compaction requested=false 2024-11-11T20:55:27,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.HRegion(2603): Flush status journal for be56906506ad5c3978d05dd159fda63b: 2024-11-11T20:55:27,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:27,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=141}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=141 2024-11-11T20:55:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=141 2024-11-11T20:55:27,805 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-11T20:55:27,805 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8040 sec 2024-11-11T20:55:27,807 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=140, table=TestAcidGuarantees in 1.8090 sec 2024-11-11T20:55:28,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=140 2024-11-11T20:55:28,146 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:55:28,147 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:55:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees 2024-11-11T20:55:28,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-11T20:55:28,148 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:55:28,149 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=142, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:55:28,149 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:55:28,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-11T20:55:28,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=143 2024-11-11T20:55:28,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:28,300 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2902): Flushing be56906506ad5c3978d05dd159fda63b 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T20:55:28,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=A 2024-11-11T20:55:28,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:28,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=B 2024-11-11T20:55:28,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:28,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=C 2024-11-11T20:55:28,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:28,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111169603aaba4ca45a493f5c93d23e5909c_be56906506ad5c3978d05dd159fda63b is 50, key is test_row_0/A:col10/1731358526505/Put/seqid=0 2024-11-11T20:55:28,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742328_1504 (size=12154) 2024-11-11T20:55:28,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-11T20:55:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:28,668 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:28,690 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:28,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358588687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:28,691 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:28,691 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:28,691 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:28,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:28,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358588689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:28,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:28,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:28,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:28,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:28,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358588690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:28,694 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:28,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:28,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:55:28,711 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111169603aaba4ca45a493f5c93d23e5909c_be56906506ad5c3978d05dd159fda63b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111169603aaba4ca45a493f5c93d23e5909c_be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:28,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/2a6658ed6ec94b5c9e06e51520ea997d, store: [table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:28,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/2a6658ed6ec94b5c9e06e51520ea997d is 175, key is test_row_0/A:col10/1731358526505/Put/seqid=0 2024-11-11T20:55:28,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742329_1505 (size=30955) 2024-11-11T20:55:28,715 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=122, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/2a6658ed6ec94b5c9e06e51520ea997d 2024-11-11T20:55:28,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/daf3b3d8437249e098e051c78ef9bd93 is 50, key is test_row_0/B:col10/1731358526505/Put/seqid=0 2024-11-11T20:55:28,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742330_1506 (size=12001) 2024-11-11T20:55:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-11T20:55:28,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:28,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358588795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:28,798 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:28,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358588795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:28,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:28,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:28,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358588805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:28,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:28,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:28,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:29,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:29,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358589005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:29,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:29,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:29,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:29,009 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:29,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358589005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:29,010 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:29,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:29,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358589015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:29,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:29,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:29,017 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:29,124 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/daf3b3d8437249e098e051c78ef9bd93 2024-11-11T20:55:29,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/294df39a445d47c88efd3d99269f4b4b is 50, key is test_row_0/C:col10/1731358526505/Put/seqid=0 2024-11-11T20:55:29,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742331_1507 (size=12001) 2024-11-11T20:55:29,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-11T20:55:29,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:29,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358589316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:29,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:29,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358589316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:29,319 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:29,319 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:29,319 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:29,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:29,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:29,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:29,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:29,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358589326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:29,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:29,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:29,328 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:29,534 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=122 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/294df39a445d47c88efd3d99269f4b4b 2024-11-11T20:55:29,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/2a6658ed6ec94b5c9e06e51520ea997d as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/2a6658ed6ec94b5c9e06e51520ea997d 2024-11-11T20:55:29,540 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/2a6658ed6ec94b5c9e06e51520ea997d, entries=150, sequenceid=122, filesize=30.2 K 2024-11-11T20:55:29,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/daf3b3d8437249e098e051c78ef9bd93 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/daf3b3d8437249e098e051c78ef9bd93 2024-11-11T20:55:29,543 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/daf3b3d8437249e098e051c78ef9bd93, entries=150, sequenceid=122, filesize=11.7 K 2024-11-11T20:55:29,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/294df39a445d47c88efd3d99269f4b4b as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/294df39a445d47c88efd3d99269f4b4b 2024-11-11T20:55:29,547 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/294df39a445d47c88efd3d99269f4b4b, entries=150, sequenceid=122, filesize=11.7 K 2024-11-11T20:55:29,547 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for be56906506ad5c3978d05dd159fda63b in 1247ms, sequenceid=122, compaction requested=true 2024-11-11T20:55:29,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.HRegion(2603): Flush status journal for be56906506ad5c3978d05dd159fda63b: 2024-11-11T20:55:29,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:29,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=143}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=143 2024-11-11T20:55:29,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=143 2024-11-11T20:55:29,550 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-11T20:55:29,550 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4000 sec 2024-11-11T20:55:29,551 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=142, table=TestAcidGuarantees in 1.4030 sec 2024-11-11T20:55:29,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:29,828 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be56906506ad5c3978d05dd159fda63b 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T20:55:29,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=A 2024-11-11T20:55:29,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:29,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=B 2024-11-11T20:55:29,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:29,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=C 2024-11-11T20:55:29,829 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:29,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111b88d966285b447399da7445525713fe_be56906506ad5c3978d05dd159fda63b is 50, key is test_row_0/A:col10/1731358529827/Put/seqid=0 2024-11-11T20:55:29,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742332_1508 (size=17234) 2024-11-11T20:55:29,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:29,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358589846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:29,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:29,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:29,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:29,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:29,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358589847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:29,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:29,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:29,853 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:29,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:29,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358589851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:29,856 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:29,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:29,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358589956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:29,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:29,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:29,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:29,959 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:29,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358589956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:29,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:29,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:29,959 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:29,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:29,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358589965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:29,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:29,967 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:29,968 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:30,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:30,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358590166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:30,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:30,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:30,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358590166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:30,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:30,167 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:30,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:30,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:30,167 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:30,178 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:30,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358590175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:30,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:30,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:30,178 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:30,238 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:55:30,241 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111b88d966285b447399da7445525713fe_be56906506ad5c3978d05dd159fda63b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111b88d966285b447399da7445525713fe_be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:30,241 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/13da9719668b46dc9374f20742f7eff7, store: [table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:30,242 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/13da9719668b46dc9374f20742f7eff7 is 175, key is test_row_0/A:col10/1731358529827/Put/seqid=0 2024-11-11T20:55:30,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742333_1509 (size=48339) 2024-11-11T20:55:30,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-11T20:55:30,295 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:55:30,296 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:55:30,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-11T20:55:30,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T20:55:30,297 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:55:30,298 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:55:30,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:55:30,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T20:55:30,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T20:55:30,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:30,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:30,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:30,450 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:30,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:30,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:30,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:30,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358590475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:30,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:30,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:30,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:30,478 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:30,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358590476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:30,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:30,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:30,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:30,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:30,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358590486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:30,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:30,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:30,487 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:30,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T20:55:30,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:30,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:30,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:30,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:30,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:30,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:30,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T20:55:30,645 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=141, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/13da9719668b46dc9374f20742f7eff7 2024-11-11T20:55:30,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/6c8028d7d1ea4c0c9733e64b65ea74d3 is 50, key is test_row_0/B:col10/1731358529827/Put/seqid=0 2024-11-11T20:55:30,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742334_1510 (size=12101) 2024-11-11T20:55:30,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/6c8028d7d1ea4c0c9733e64b65ea74d3 2024-11-11T20:55:30,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/5833fbae9f34433aa351ff512200f55d is 50, key is test_row_0/C:col10/1731358529827/Put/seqid=0 2024-11-11T20:55:30,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742335_1511 (size=12101) 2024-11-11T20:55:30,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T20:55:30,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:30,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:30,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:30,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:30,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:30,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:30,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T20:55:30,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:30,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:30,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:30,907 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:30,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:30,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:30,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T20:55:30,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358590985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:30,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:30,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358590986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:30,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:30,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:30,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:30,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:30,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:30,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358590996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:30,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:30,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:30,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:31,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T20:55:31,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:31,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:31,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:31,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:31,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:31,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:31,063 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=141 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/5833fbae9f34433aa351ff512200f55d 2024-11-11T20:55:31,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/13da9719668b46dc9374f20742f7eff7 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/13da9719668b46dc9374f20742f7eff7 2024-11-11T20:55:31,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/13da9719668b46dc9374f20742f7eff7, entries=250, sequenceid=141, filesize=47.2 K 2024-11-11T20:55:31,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/6c8028d7d1ea4c0c9733e64b65ea74d3 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/6c8028d7d1ea4c0c9733e64b65ea74d3 2024-11-11T20:55:31,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/6c8028d7d1ea4c0c9733e64b65ea74d3, entries=150, sequenceid=141, filesize=11.8 K 2024-11-11T20:55:31,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/5833fbae9f34433aa351ff512200f55d as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/5833fbae9f34433aa351ff512200f55d 2024-11-11T20:55:31,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/5833fbae9f34433aa351ff512200f55d, entries=150, sequenceid=141, filesize=11.8 K 2024-11-11T20:55:31,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for be56906506ad5c3978d05dd159fda63b in 1248ms, sequenceid=141, compaction requested=true 2024-11-11T20:55:31,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be56906506ad5c3978d05dd159fda63b: 2024-11-11T20:55:31,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be56906506ad5c3978d05dd159fda63b:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:55:31,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:31,077 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:55:31,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be56906506ad5c3978d05dd159fda63b:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:55:31,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:55:31,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be56906506ad5c3978d05dd159fda63b:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:55:31,077 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:55:31,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:55:31,078 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:55:31,078 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48310 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:55:31,078 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): be56906506ad5c3978d05dd159fda63b/A is initiating minor compaction (all files) 2024-11-11T20:55:31,078 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): be56906506ad5c3978d05dd159fda63b/B is initiating minor compaction (all files) 2024-11-11T20:55:31,078 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be56906506ad5c3978d05dd159fda63b/B in TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:31,078 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be56906506ad5c3978d05dd159fda63b/A in TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:31,078 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/05fdc3c97d6e46b8ac605e7001012451, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/d596b833e82541cca9b4d06cbe719152, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/2a6658ed6ec94b5c9e06e51520ea997d, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/13da9719668b46dc9374f20742f7eff7] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp, totalSize=138.1 K 2024-11-11T20:55:31,078 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/54e9eb0337b4414cb5a6192394f8794d, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/bf971f5dd91645eca4045d02ed96a8d9, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/daf3b3d8437249e098e051c78ef9bd93, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/6c8028d7d1ea4c0c9733e64b65ea74d3] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp, totalSize=47.2 K 2024-11-11T20:55:31,078 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:31,078 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. files: [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/05fdc3c97d6e46b8ac605e7001012451, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/d596b833e82541cca9b4d06cbe719152, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/2a6658ed6ec94b5c9e06e51520ea997d, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/13da9719668b46dc9374f20742f7eff7] 2024-11-11T20:55:31,078 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 54e9eb0337b4414cb5a6192394f8794d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731358523172 2024-11-11T20:55:31,078 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05fdc3c97d6e46b8ac605e7001012451, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731358523172 2024-11-11T20:55:31,079 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting bf971f5dd91645eca4045d02ed96a8d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731358524336 2024-11-11T20:55:31,079 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting d596b833e82541cca9b4d06cbe719152, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731358524336 2024-11-11T20:55:31,079 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting daf3b3d8437249e098e051c78ef9bd93, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731358526505 2024-11-11T20:55:31,079 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a6658ed6ec94b5c9e06e51520ea997d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731358526505 2024-11-11T20:55:31,079 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c8028d7d1ea4c0c9733e64b65ea74d3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731358528671 2024-11-11T20:55:31,079 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13da9719668b46dc9374f20742f7eff7, keycount=250, bloomtype=ROW, size=47.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731358528671 2024-11-11T20:55:31,094 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:31,096 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111179635925b4bf4b6fb87c46aceaa0e9f0_be56906506ad5c3978d05dd159fda63b store=[table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:31,096 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be56906506ad5c3978d05dd159fda63b#B#compaction#430 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:55:31,097 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/0f44b67a11d2414ba6d9f91d8644104a is 50, key is test_row_0/B:col10/1731358529827/Put/seqid=0 2024-11-11T20:55:31,099 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111179635925b4bf4b6fb87c46aceaa0e9f0_be56906506ad5c3978d05dd159fda63b, store=[table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:31,099 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111179635925b4bf4b6fb87c46aceaa0e9f0_be56906506ad5c3978d05dd159fda63b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:31,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742337_1513 (size=4469) 2024-11-11T20:55:31,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742336_1512 (size=12443) 2024-11-11T20:55:31,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-11T20:55:31,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:31,213 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing be56906506ad5c3978d05dd159fda63b 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T20:55:31,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=A 2024-11-11T20:55:31,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:31,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=B 2024-11-11T20:55:31,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:31,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=C 2024-11-11T20:55:31,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:31,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111123288af8ee3e422e99dd5e749c8e0a51_be56906506ad5c3978d05dd159fda63b is 50, key is test_row_0/A:col10/1731358529845/Put/seqid=0 2024-11-11T20:55:31,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742338_1514 (size=12304) 2024-11-11T20:55:31,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T20:55:31,506 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be56906506ad5c3978d05dd159fda63b#A#compaction#429 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:55:31,506 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/a690425599d944e9a93f90a162ab7e67 is 175, key is test_row_0/A:col10/1731358529827/Put/seqid=0 2024-11-11T20:55:31,511 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/0f44b67a11d2414ba6d9f91d8644104a as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/0f44b67a11d2414ba6d9f91d8644104a 2024-11-11T20:55:31,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742339_1515 (size=31397) 2024-11-11T20:55:31,521 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in be56906506ad5c3978d05dd159fda63b/B of be56906506ad5c3978d05dd159fda63b into 0f44b67a11d2414ba6d9f91d8644104a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:55:31,521 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be56906506ad5c3978d05dd159fda63b: 2024-11-11T20:55:31,521 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., storeName=be56906506ad5c3978d05dd159fda63b/B, priority=12, startTime=1731358531077; duration=0sec 2024-11-11T20:55:31,521 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:55:31,521 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be56906506ad5c3978d05dd159fda63b:B 2024-11-11T20:55:31,521 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T20:55:31,522 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48310 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T20:55:31,522 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): be56906506ad5c3978d05dd159fda63b/C is initiating minor compaction (all files) 2024-11-11T20:55:31,522 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be56906506ad5c3978d05dd159fda63b/C in TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:31,522 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/1fa941b936924ed58addf5524a26b5f5, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/a9cab78c72af4540b644d29506c4ebe3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/294df39a445d47c88efd3d99269f4b4b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/5833fbae9f34433aa351ff512200f55d] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp, totalSize=47.2 K 2024-11-11T20:55:31,522 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fa941b936924ed58addf5524a26b5f5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731358523172 2024-11-11T20:55:31,523 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting a9cab78c72af4540b644d29506c4ebe3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731358524336 2024-11-11T20:55:31,523 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 294df39a445d47c88efd3d99269f4b4b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=122, earliestPutTs=1731358526505 2024-11-11T20:55:31,523 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 5833fbae9f34433aa351ff512200f55d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731358528671 2024-11-11T20:55:31,530 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be56906506ad5c3978d05dd159fda63b#C#compaction#432 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:55:31,530 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/d1398877a709405d8b7922a93c56dcd0 is 50, key is test_row_0/C:col10/1731358529827/Put/seqid=0 2024-11-11T20:55:31,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742340_1516 (size=12443) 2024-11-11T20:55:31,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:55:31,626 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111123288af8ee3e422e99dd5e749c8e0a51_be56906506ad5c3978d05dd159fda63b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111123288af8ee3e422e99dd5e749c8e0a51_be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:31,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/6e5b69024248455abfc3d96287fbdc7c, store: [table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:31,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/6e5b69024248455abfc3d96287fbdc7c is 175, key is test_row_0/A:col10/1731358529845/Put/seqid=0 2024-11-11T20:55:31,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742341_1517 (size=31105) 2024-11-11T20:55:31,921 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/a690425599d944e9a93f90a162ab7e67 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/a690425599d944e9a93f90a162ab7e67 2024-11-11T20:55:31,925 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in be56906506ad5c3978d05dd159fda63b/A of be56906506ad5c3978d05dd159fda63b into a690425599d944e9a93f90a162ab7e67(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:55:31,925 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be56906506ad5c3978d05dd159fda63b: 2024-11-11T20:55:31,925 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., storeName=be56906506ad5c3978d05dd159fda63b/A, priority=12, startTime=1731358531077; duration=0sec 2024-11-11T20:55:31,925 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:31,925 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be56906506ad5c3978d05dd159fda63b:A 2024-11-11T20:55:31,937 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/d1398877a709405d8b7922a93c56dcd0 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/d1398877a709405d8b7922a93c56dcd0 2024-11-11T20:55:31,940 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in be56906506ad5c3978d05dd159fda63b/C of be56906506ad5c3978d05dd159fda63b into d1398877a709405d8b7922a93c56dcd0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:55:31,941 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be56906506ad5c3978d05dd159fda63b: 2024-11-11T20:55:31,941 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., storeName=be56906506ad5c3978d05dd159fda63b/C, priority=12, startTime=1731358531077; duration=0sec 2024-11-11T20:55:31,941 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:31,941 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be56906506ad5c3978d05dd159fda63b:C 2024-11-11T20:55:31,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:31,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:32,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:32,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358592018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:32,022 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:32,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:32,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358592020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:32,023 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:32,023 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:32,023 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:32,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:32,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358592022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:32,026 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:32,031 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=154, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/6e5b69024248455abfc3d96287fbdc7c 2024-11-11T20:55:32,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/bf7da0749f90472cb96e5104162f36ed is 50, key is test_row_0/B:col10/1731358529845/Put/seqid=0 2024-11-11T20:55:32,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742342_1518 (size=12151) 2024-11-11T20:55:32,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:32,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358592125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:32,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:32,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:32,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358592125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:32,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:32,128 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:32,128 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:32,128 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:32,128 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:32,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:32,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358592136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:32,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:32,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:32,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358592335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:32,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:32,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:32,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:32,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:32,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358592335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:32,338 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:32,338 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:32,338 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:32,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:32,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358592345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:32,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:32,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:32,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:32,441 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/bf7da0749f90472cb96e5104162f36ed 2024-11-11T20:55:32,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T20:55:32,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/af8b839743bf448b93c668c91fa0b1f4 is 50, key is test_row_0/C:col10/1731358529845/Put/seqid=0 2024-11-11T20:55:32,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742343_1519 (size=12151) 2024-11-11T20:55:32,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:32,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358592646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:32,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:32,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:32,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:32,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:32,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358592646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:32,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:32,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:32,649 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:32,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:32,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358592656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:32,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:32,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:32,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:32,855 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/af8b839743bf448b93c668c91fa0b1f4 2024-11-11T20:55:32,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/6e5b69024248455abfc3d96287fbdc7c as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/6e5b69024248455abfc3d96287fbdc7c 2024-11-11T20:55:32,861 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/6e5b69024248455abfc3d96287fbdc7c, entries=150, sequenceid=154, filesize=30.4 K 2024-11-11T20:55:32,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/bf7da0749f90472cb96e5104162f36ed as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/bf7da0749f90472cb96e5104162f36ed 2024-11-11T20:55:32,865 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/bf7da0749f90472cb96e5104162f36ed, entries=150, sequenceid=154, filesize=11.9 K 2024-11-11T20:55:32,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/af8b839743bf448b93c668c91fa0b1f4 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/af8b839743bf448b93c668c91fa0b1f4 2024-11-11T20:55:32,868 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/af8b839743bf448b93c668c91fa0b1f4, entries=150, sequenceid=154, filesize=11.9 K 2024-11-11T20:55:32,868 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for be56906506ad5c3978d05dd159fda63b in 1656ms, sequenceid=154, compaction requested=false 2024-11-11T20:55:32,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for be56906506ad5c3978d05dd159fda63b: 2024-11-11T20:55:32,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:32,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-11T20:55:32,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-11T20:55:32,871 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-11T20:55:32,871 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5720 sec 2024-11-11T20:55:32,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 2.5760 sec 2024-11-11T20:55:33,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:33,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be56906506ad5c3978d05dd159fda63b 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T20:55:33,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=A 2024-11-11T20:55:33,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:33,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=B 2024-11-11T20:55:33,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:33,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=C 2024-11-11T20:55:33,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:33,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118be64410a2e841e29a2f4fb6c6e9fb57_be56906506ad5c3978d05dd159fda63b is 50, key is test_row_0/A:col10/1731358532021/Put/seqid=0 2024-11-11T20:55:33,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742344_1520 (size=14794) 2024-11-11T20:55:33,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:33,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358593170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:33,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:33,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358593172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:33,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:33,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358593173, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:33,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:33,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:33,176 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:33,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:33,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358593286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:33,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:33,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358593286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:33,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:33,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358593286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:33,289 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:33,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358593496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:33,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358593496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:33,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:33,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358593496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:33,569 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:55:33,571 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118be64410a2e841e29a2f4fb6c6e9fb57_be56906506ad5c3978d05dd159fda63b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118be64410a2e841e29a2f4fb6c6e9fb57_be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:33,572 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/ce22d93a65ec4fd7b860d8c57a62c334, store: [table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:33,573 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/ce22d93a65ec4fd7b860d8c57a62c334 is 175, key is test_row_0/A:col10/1731358532021/Put/seqid=0 2024-11-11T20:55:33,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742345_1521 (size=39749) 2024-11-11T20:55:33,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:33,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358593805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:33,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:33,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:33,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:33,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:33,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358593806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:33,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:33,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:33,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358593806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:33,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:33,809 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:33,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:33,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:33,809 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:33,976 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=177, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/ce22d93a65ec4fd7b860d8c57a62c334 2024-11-11T20:55:33,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/6e9a481cdea34b5980403885c2c5e711 is 50, key is test_row_0/B:col10/1731358532021/Put/seqid=0 2024-11-11T20:55:33,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742346_1522 (size=12151) 2024-11-11T20:55:34,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:34,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358594315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:34,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:34,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:34,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358594315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:34,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:34,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:34,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:34,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:34,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:34,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358594315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:34,316 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:34,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:34,316 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:34,317 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:34,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/6e9a481cdea34b5980403885c2c5e711 2024-11-11T20:55:34,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/c085d32529bd4af096b8dd60efd0b524 is 50, key is test_row_0/C:col10/1731358532021/Put/seqid=0 2024-11-11T20:55:34,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742347_1523 (size=12151) 2024-11-11T20:55:34,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=177 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/c085d32529bd4af096b8dd60efd0b524 2024-11-11T20:55:34,405 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/ce22d93a65ec4fd7b860d8c57a62c334 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/ce22d93a65ec4fd7b860d8c57a62c334 2024-11-11T20:55:34,408 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/ce22d93a65ec4fd7b860d8c57a62c334, entries=200, sequenceid=177, filesize=38.8 K 2024-11-11T20:55:34,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/6e9a481cdea34b5980403885c2c5e711 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/6e9a481cdea34b5980403885c2c5e711 2024-11-11T20:55:34,411 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/6e9a481cdea34b5980403885c2c5e711, entries=150, sequenceid=177, filesize=11.9 K 2024-11-11T20:55:34,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/c085d32529bd4af096b8dd60efd0b524 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/c085d32529bd4af096b8dd60efd0b524 2024-11-11T20:55:34,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/c085d32529bd4af096b8dd60efd0b524, entries=150, sequenceid=177, filesize=11.9 K 2024-11-11T20:55:34,415 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for be56906506ad5c3978d05dd159fda63b in 1257ms, sequenceid=177, compaction requested=true 2024-11-11T20:55:34,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be56906506ad5c3978d05dd159fda63b: 2024-11-11T20:55:34,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be56906506ad5c3978d05dd159fda63b:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:55:34,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:34,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be56906506ad5c3978d05dd159fda63b:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:55:34,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:34,415 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:55:34,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be56906506ad5c3978d05dd159fda63b:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:55:34,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:55:34,415 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:55:34,416 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36745 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:55:34,416 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102251 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:55:34,416 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): be56906506ad5c3978d05dd159fda63b/B is initiating minor compaction (all files) 2024-11-11T20:55:34,416 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): be56906506ad5c3978d05dd159fda63b/A is initiating minor compaction (all files) 2024-11-11T20:55:34,416 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be56906506ad5c3978d05dd159fda63b/A in TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:34,416 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be56906506ad5c3978d05dd159fda63b/B in TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:34,416 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/a690425599d944e9a93f90a162ab7e67, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/6e5b69024248455abfc3d96287fbdc7c, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/ce22d93a65ec4fd7b860d8c57a62c334] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp, totalSize=99.9 K 2024-11-11T20:55:34,416 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/0f44b67a11d2414ba6d9f91d8644104a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/bf7da0749f90472cb96e5104162f36ed, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/6e9a481cdea34b5980403885c2c5e711] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp, totalSize=35.9 K 2024-11-11T20:55:34,416 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:34,416 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. files: [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/a690425599d944e9a93f90a162ab7e67, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/6e5b69024248455abfc3d96287fbdc7c, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/ce22d93a65ec4fd7b860d8c57a62c334] 2024-11-11T20:55:34,416 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f44b67a11d2414ba6d9f91d8644104a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731358528671 2024-11-11T20:55:34,416 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting a690425599d944e9a93f90a162ab7e67, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731358528671 2024-11-11T20:55:34,417 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting bf7da0749f90472cb96e5104162f36ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731358529845 2024-11-11T20:55:34,417 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e5b69024248455abfc3d96287fbdc7c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731358529845 2024-11-11T20:55:34,417 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e9a481cdea34b5980403885c2c5e711, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731358532014 2024-11-11T20:55:34,417 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce22d93a65ec4fd7b860d8c57a62c334, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731358532014 2024-11-11T20:55:34,422 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be56906506ad5c3978d05dd159fda63b#B#compaction#438 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:55:34,423 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/871e48d60e35440095992c6cdae2c202 is 50, key is test_row_0/B:col10/1731358532021/Put/seqid=0 2024-11-11T20:55:34,423 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:34,431 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111058ebfb7c5da49a8ab70a875a00ffc71_be56906506ad5c3978d05dd159fda63b store=[table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:34,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742348_1524 (size=12595) 2024-11-11T20:55:34,433 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111058ebfb7c5da49a8ab70a875a00ffc71_be56906506ad5c3978d05dd159fda63b, store=[table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:34,433 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111058ebfb7c5da49a8ab70a875a00ffc71_be56906506ad5c3978d05dd159fda63b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:34,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742349_1525 (size=4469) 2024-11-11T20:55:34,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-11T20:55:34,456 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:55:34,456 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:55:34,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-11T20:55:34,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T20:55:34,458 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:55:34,458 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:55:34,458 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:55:34,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T20:55:34,610 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-11T20:55:34,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:34,610 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing be56906506ad5c3978d05dd159fda63b 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T20:55:34,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=A 2024-11-11T20:55:34,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:34,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=B 2024-11-11T20:55:34,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:34,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=C 2024-11-11T20:55:34,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:34,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111161503d67288e49fda9f798738b1e5f2d_be56906506ad5c3978d05dd159fda63b is 50, key is test_row_0/A:col10/1731358533172/Put/seqid=0 2024-11-11T20:55:34,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742350_1526 (size=12304) 2024-11-11T20:55:34,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T20:55:34,837 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/871e48d60e35440095992c6cdae2c202 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/871e48d60e35440095992c6cdae2c202 2024-11-11T20:55:34,838 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be56906506ad5c3978d05dd159fda63b#A#compaction#439 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:55:34,838 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/5e643a6043874604b93e807fce833174 is 175, key is test_row_0/A:col10/1731358532021/Put/seqid=0 2024-11-11T20:55:34,841 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be56906506ad5c3978d05dd159fda63b/B of be56906506ad5c3978d05dd159fda63b into 871e48d60e35440095992c6cdae2c202(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:55:34,841 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be56906506ad5c3978d05dd159fda63b: 2024-11-11T20:55:34,841 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., storeName=be56906506ad5c3978d05dd159fda63b/B, priority=13, startTime=1731358534415; duration=0sec 2024-11-11T20:55:34,841 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:55:34,841 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be56906506ad5c3978d05dd159fda63b:B 2024-11-11T20:55:34,841 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:55:34,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742351_1527 (size=31549) 2024-11-11T20:55:34,844 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36745 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:55:34,844 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): be56906506ad5c3978d05dd159fda63b/C is initiating minor compaction (all files) 2024-11-11T20:55:34,844 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be56906506ad5c3978d05dd159fda63b/C in TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:34,844 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/d1398877a709405d8b7922a93c56dcd0, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/af8b839743bf448b93c668c91fa0b1f4, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/c085d32529bd4af096b8dd60efd0b524] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp, totalSize=35.9 K 2024-11-11T20:55:34,844 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting d1398877a709405d8b7922a93c56dcd0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=141, earliestPutTs=1731358528671 2024-11-11T20:55:34,845 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting af8b839743bf448b93c668c91fa0b1f4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1731358529845 2024-11-11T20:55:34,845 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting c085d32529bd4af096b8dd60efd0b524, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731358532014 2024-11-11T20:55:34,848 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/5e643a6043874604b93e807fce833174 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/5e643a6043874604b93e807fce833174 2024-11-11T20:55:34,853 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be56906506ad5c3978d05dd159fda63b/A of be56906506ad5c3978d05dd159fda63b into 5e643a6043874604b93e807fce833174(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:55:34,853 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be56906506ad5c3978d05dd159fda63b: 2024-11-11T20:55:34,853 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., storeName=be56906506ad5c3978d05dd159fda63b/A, priority=13, startTime=1731358534415; duration=0sec 2024-11-11T20:55:34,853 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:34,853 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be56906506ad5c3978d05dd159fda63b:A 2024-11-11T20:55:34,854 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be56906506ad5c3978d05dd159fda63b#C#compaction#441 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:55:34,855 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/97bf0fa267d84efd848c534ae6eadce2 is 50, key is test_row_0/C:col10/1731358532021/Put/seqid=0 2024-11-11T20:55:34,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742352_1528 (size=12595) 2024-11-11T20:55:35,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:55:35,024 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111161503d67288e49fda9f798738b1e5f2d_be56906506ad5c3978d05dd159fda63b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111161503d67288e49fda9f798738b1e5f2d_be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:35,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/858f956f7b344815b4e96d7bb96bf890, store: [table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:35,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/858f956f7b344815b4e96d7bb96bf890 is 175, key is test_row_0/A:col10/1731358533172/Put/seqid=0 2024-11-11T20:55:35,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742353_1529 (size=31105) 2024-11-11T20:55:35,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T20:55:35,266 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/97bf0fa267d84efd848c534ae6eadce2 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/97bf0fa267d84efd848c534ae6eadce2 2024-11-11T20:55:35,270 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be56906506ad5c3978d05dd159fda63b/C of be56906506ad5c3978d05dd159fda63b into 97bf0fa267d84efd848c534ae6eadce2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:55:35,270 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be56906506ad5c3978d05dd159fda63b: 2024-11-11T20:55:35,270 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., storeName=be56906506ad5c3978d05dd159fda63b/C, priority=13, startTime=1731358534415; duration=0sec 2024-11-11T20:55:35,270 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:35,270 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be56906506ad5c3978d05dd159fda63b:C 2024-11-11T20:55:35,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:35,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:35,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:35,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358595351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:35,356 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:35,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:35,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358595355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:35,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:35,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:35,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358595356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:35,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:35,360 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:35,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:35,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:35,360 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:35,431 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=189, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/858f956f7b344815b4e96d7bb96bf890 2024-11-11T20:55:35,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/e4189b68d07348bead8d965c8a87ae8d is 50, key is test_row_0/B:col10/1731358533172/Put/seqid=0 2024-11-11T20:55:35,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742354_1530 (size=12151) 2024-11-11T20:55:35,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:35,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358595465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:35,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:35,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358595465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:35,466 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:35,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:35,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358595466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:35,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:35,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T20:55:35,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:35,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358595675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:35,678 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:35,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:35,678 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:35,678 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:35,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358595675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:35,678 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:35,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:35,678 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:35,678 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:35,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358595675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:35,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:35,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:35,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:35,841 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/e4189b68d07348bead8d965c8a87ae8d 2024-11-11T20:55:35,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/25579a69e8db4e2c9f397668b529aec8 is 50, key is test_row_0/C:col10/1731358533172/Put/seqid=0 2024-11-11T20:55:35,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742355_1531 (size=12151) 2024-11-11T20:55:35,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:35,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48744 deadline: 1731358595956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:35,958 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_0' in be56906506ad5c3978d05dd159fda63b of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18277 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:55:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:35,958 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:35,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48786 deadline: 1731358595986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:35,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358595986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:35,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358595986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:35,989 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 51ca66f7ee3c,44669,1731358362066 for 'test_row_2' in be56906506ad5c3978d05dd159fda63b of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18309 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor42.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T20:55:35,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358595986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:35,989 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:35,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:35,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:35,990 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:36,250 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/25579a69e8db4e2c9f397668b529aec8 2024-11-11T20:55:36,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/858f956f7b344815b4e96d7bb96bf890 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/858f956f7b344815b4e96d7bb96bf890 2024-11-11T20:55:36,258 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/858f956f7b344815b4e96d7bb96bf890, entries=150, sequenceid=189, filesize=30.4 K 2024-11-11T20:55:36,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/e4189b68d07348bead8d965c8a87ae8d as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/e4189b68d07348bead8d965c8a87ae8d 2024-11-11T20:55:36,262 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/e4189b68d07348bead8d965c8a87ae8d, entries=150, sequenceid=189, filesize=11.9 K 2024-11-11T20:55:36,262 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/25579a69e8db4e2c9f397668b529aec8 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/25579a69e8db4e2c9f397668b529aec8 2024-11-11T20:55:36,265 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/25579a69e8db4e2c9f397668b529aec8, entries=150, sequenceid=189, filesize=11.9 K 2024-11-11T20:55:36,265 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for be56906506ad5c3978d05dd159fda63b in 1655ms, sequenceid=189, compaction requested=false 2024-11-11T20:55:36,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for be56906506ad5c3978d05dd159fda63b: 2024-11-11T20:55:36,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:36,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-11T20:55:36,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-11T20:55:36,268 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-11T20:55:36,268 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8090 sec 2024-11-11T20:55:36,270 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.8130 sec 2024-11-11T20:55:36,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:36,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing be56906506ad5c3978d05dd159fda63b 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-11T20:55:36,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=A 2024-11-11T20:55:36,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:55:36,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=B 2024-11-11T20:55:36,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:55:36,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=C 2024-11-11T20:55:36,498 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T20:55:36,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ce4fc5bef02b400c82d91bbc16b0f831_be56906506ad5c3978d05dd159fda63b is 50, key is test_row_0/A:col10/1731358536497/Put/seqid=0 2024-11-11T20:55:36,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742356_1532 (size=14794) 2024-11-11T20:55:36,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:36,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358596508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:36,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:36,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:36,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:36,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358596509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:36,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:36,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358596509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:36,513 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:36,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-11T20:55:36,606 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:55:36,607 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T20:55:36,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-11T20:55:36,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T20:55:36,608 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T20:55:36,609 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T20:55:36,609 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T20:55:36,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:36,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358596615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:36,618 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:36,618 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:36,618 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:36,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:36,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358596615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:36,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:36,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358596616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:36,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:36,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T20:55:36,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T20:55:36,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:36,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:36,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:36,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:36,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:36,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:36,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358596825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:36,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:36,827 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:36,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358596826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:36,827 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:36,827 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:36,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:36,828 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:36,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:36,828 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358596826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:36,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:36,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:36,828 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:36,907 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:55:36,910 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ce4fc5bef02b400c82d91bbc16b0f831_be56906506ad5c3978d05dd159fda63b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ce4fc5bef02b400c82d91bbc16b0f831_be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:36,911 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/25504385f41442a598985c29de1bb566, store: [table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:36,911 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/25504385f41442a598985c29de1bb566 is 175, key is test_row_0/A:col10/1731358536497/Put/seqid=0 2024-11-11T20:55:36,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T20:55:36,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:36,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:36,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:36,913 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:36,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:36,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:36,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742357_1533 (size=39749) 2024-11-11T20:55:36,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T20:55:37,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T20:55:37,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:37,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:37,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:37,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:37,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:37,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:37,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358597136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:37,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:37,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358597136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:37,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:37,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358597136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:37,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T20:55:37,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:37,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:37,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:37,218 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:37,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:37,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:37,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T20:55:37,317 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=213, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/25504385f41442a598985c29de1bb566 2024-11-11T20:55:37,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/ebd22342e8944a8eb79343f59c678af9 is 50, key is test_row_0/B:col10/1731358536497/Put/seqid=0 2024-11-11T20:55:37,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742358_1534 (size=12151) 2024-11-11T20:55:37,370 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T20:55:37,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:37,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:37,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:37,370 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:37,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:37,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:37,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T20:55:37,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:37,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:37,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:37,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:37,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:37,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:37,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:37,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48792 deadline: 1731358597646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:37,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:37,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:37,648 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:37,650 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:37,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T20:55:37,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48734 deadline: 1731358597646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:37,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=44669 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:48802 deadline: 1731358597646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:37,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:37,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 , the old value is region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:37,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:37,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T20:55:37,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:37,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., hostname=51ca66f7ee3c,44669,1731358362066, seqNum=5 because the exception is null or not the one we care about 2024-11-11T20:55:37,651 INFO [Thread-2142 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2142. 2024-11-11T20:55:37,651 INFO [Thread-2146 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2146. 2024-11-11T20:55:37,651 DEBUG [Thread-2142 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:55:37,651 DEBUG [Thread-2146 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:55:37,651 DEBUG [Thread-2146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:37,651 DEBUG [Thread-2142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:37,651 DEBUG [Thread-2146 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:37,651 DEBUG [Thread-2142 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:37,651 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:55:37,651 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:55:37,653 INFO [Thread-2148 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2148. 2024-11-11T20:55:37,653 DEBUG [Thread-2148 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:55:37,653 DEBUG [Thread-2148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:37,653 DEBUG [Thread-2148 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:37,653 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:55:37,653 INFO [Thread-2144 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2144. 2024-11-11T20:55:37,653 DEBUG [Thread-2144 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:55:37,653 DEBUG [Thread-2144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:37,653 DEBUG [Thread-2144 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:37,653 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:55:37,654 INFO [Thread-2150 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2150. 2024-11-11T20:55:37,654 DEBUG [Thread-2150 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:55:37,654 DEBUG [Thread-2150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:37,654 DEBUG [Thread-2150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:37,654 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:55:37,675 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T20:55:37,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:37,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:37,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:37,676 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:37,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:37,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:37,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/ebd22342e8944a8eb79343f59c678af9 2024-11-11T20:55:37,740 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/e9463852ecaa4150936733e339613637 is 50, key is test_row_0/C:col10/1731358536497/Put/seqid=0 2024-11-11T20:55:37,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742359_1535 (size=12151) 2024-11-11T20:55:37,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T20:55:37,829 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T20:55:37,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:37,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:37,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:37,829 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:37,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:37,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T20:55:37,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:37,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:37,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:37,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:37,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:37,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:38,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T20:55:38,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:38,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:38,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:38,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] handler.RSProcedureHandler(58): pid=149 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:38,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-0 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=149 java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:38,145 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=213 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/e9463852ecaa4150936733e339613637 2024-11-11T20:55:38,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4177): Remote procedure failed, pid=149 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T20:55:38,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/25504385f41442a598985c29de1bb566 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/25504385f41442a598985c29de1bb566 2024-11-11T20:55:38,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/25504385f41442a598985c29de1bb566, entries=200, sequenceid=213, filesize=38.8 K 2024-11-11T20:55:38,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/ebd22342e8944a8eb79343f59c678af9 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/ebd22342e8944a8eb79343f59c678af9 2024-11-11T20:55:38,163 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/ebd22342e8944a8eb79343f59c678af9, entries=150, sequenceid=213, filesize=11.9 K 2024-11-11T20:55:38,163 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/e9463852ecaa4150936733e339613637 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/e9463852ecaa4150936733e339613637 2024-11-11T20:55:38,167 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/e9463852ecaa4150936733e339613637, entries=150, sequenceid=213, filesize=11.9 K 2024-11-11T20:55:38,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for be56906506ad5c3978d05dd159fda63b in 1669ms, sequenceid=213, compaction requested=true 2024-11-11T20:55:38,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for be56906506ad5c3978d05dd159fda63b: 2024-11-11T20:55:38,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be56906506ad5c3978d05dd159fda63b:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T20:55:38,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:38,168 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:55:38,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be56906506ad5c3978d05dd159fda63b:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T20:55:38,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:38,168 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:55:38,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store be56906506ad5c3978d05dd159fda63b:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T20:55:38,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:55:38,168 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:55:38,168 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:55:38,168 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1541): be56906506ad5c3978d05dd159fda63b/A is initiating minor compaction (all files) 2024-11-11T20:55:38,168 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): be56906506ad5c3978d05dd159fda63b/B is initiating minor compaction (all files) 2024-11-11T20:55:38,168 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be56906506ad5c3978d05dd159fda63b/A in TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:38,168 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be56906506ad5c3978d05dd159fda63b/B in TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:38,168 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/5e643a6043874604b93e807fce833174, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/858f956f7b344815b4e96d7bb96bf890, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/25504385f41442a598985c29de1bb566] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp, totalSize=100.0 K 2024-11-11T20:55:38,168 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:38,168 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/871e48d60e35440095992c6cdae2c202, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/e4189b68d07348bead8d965c8a87ae8d, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/ebd22342e8944a8eb79343f59c678af9] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp, totalSize=36.0 K 2024-11-11T20:55:38,169 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. files: [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/5e643a6043874604b93e807fce833174, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/858f956f7b344815b4e96d7bb96bf890, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/25504385f41442a598985c29de1bb566] 2024-11-11T20:55:38,169 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 871e48d60e35440095992c6cdae2c202, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731358532014 2024-11-11T20:55:38,169 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e643a6043874604b93e807fce833174, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731358532014 2024-11-11T20:55:38,169 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 858f956f7b344815b4e96d7bb96bf890, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731358533165 2024-11-11T20:55:38,169 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4189b68d07348bead8d965c8a87ae8d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731358533165 2024-11-11T20:55:38,169 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] compactions.Compactor(225): Compacting 25504385f41442a598985c29de1bb566, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731358535350 2024-11-11T20:55:38,169 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebd22342e8944a8eb79343f59c678af9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731358535350 2024-11-11T20:55:38,173 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:38,174 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be56906506ad5c3978d05dd159fda63b#B#compaction#447 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:55:38,174 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111936a67945cec4c7eb7536b21f5dbef92_be56906506ad5c3978d05dd159fda63b store=[table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:38,174 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/12f359bda0bc4484981d522bb0364f1e is 50, key is test_row_0/B:col10/1731358536497/Put/seqid=0 2024-11-11T20:55:38,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742360_1536 (size=12697) 2024-11-11T20:55:38,178 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111936a67945cec4c7eb7536b21f5dbef92_be56906506ad5c3978d05dd159fda63b, store=[table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:38,178 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111936a67945cec4c7eb7536b21f5dbef92_be56906506ad5c3978d05dd159fda63b because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:38,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742361_1537 (size=4469) 2024-11-11T20:55:38,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44669 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-11T20:55:38,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:38,299 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing be56906506ad5c3978d05dd159fda63b 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T20:55:38,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=A 2024-11-11T20:55:38,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:38,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=B 2024-11-11T20:55:38,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:38,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=C 2024-11-11T20:55:38,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:38,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111eb92f4abb404bdf99423c23f15fbeb8_be56906506ad5c3978d05dd159fda63b is 50, key is test_row_0/A:col10/1731358536503/Put/seqid=0 2024-11-11T20:55:38,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742362_1538 (size=12304) 2024-11-11T20:55:38,584 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): be56906506ad5c3978d05dd159fda63b#A#compaction#448 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:55:38,586 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/95f69896a8f84deca43bbd50bc66f881 is 175, key is test_row_0/A:col10/1731358536497/Put/seqid=0 2024-11-11T20:55:38,589 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/12f359bda0bc4484981d522bb0364f1e as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/12f359bda0bc4484981d522bb0364f1e 2024-11-11T20:55:38,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742363_1539 (size=31651) 2024-11-11T20:55:38,593 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be56906506ad5c3978d05dd159fda63b/B of be56906506ad5c3978d05dd159fda63b into 12f359bda0bc4484981d522bb0364f1e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:55:38,593 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be56906506ad5c3978d05dd159fda63b: 2024-11-11T20:55:38,593 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., storeName=be56906506ad5c3978d05dd159fda63b/B, priority=13, startTime=1731358538168; duration=0sec 2024-11-11T20:55:38,594 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T20:55:38,594 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be56906506ad5c3978d05dd159fda63b:B 2024-11-11T20:55:38,594 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T20:55:38,595 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T20:55:38,595 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1541): be56906506ad5c3978d05dd159fda63b/C is initiating minor compaction (all files) 2024-11-11T20:55:38,595 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of be56906506ad5c3978d05dd159fda63b/C in TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:38,595 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/97bf0fa267d84efd848c534ae6eadce2, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/25579a69e8db4e2c9f397668b529aec8, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/e9463852ecaa4150936733e339613637] into tmpdir=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp, totalSize=36.0 K 2024-11-11T20:55:38,595 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 97bf0fa267d84efd848c534ae6eadce2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=177, earliestPutTs=1731358532014 2024-11-11T20:55:38,596 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting 25579a69e8db4e2c9f397668b529aec8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731358533165 2024-11-11T20:55:38,596 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9463852ecaa4150936733e339613637, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=213, earliestPutTs=1731358535350 2024-11-11T20:55:38,604 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): be56906506ad5c3978d05dd159fda63b#C#compaction#450 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T20:55:38,604 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/e7fecfb1f7a44592b62cfe852f4e2d45 is 50, key is test_row_0/C:col10/1731358536497/Put/seqid=0 2024-11-11T20:55:38,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742364_1540 (size=12697) 2024-11-11T20:55:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=44669 {}] regionserver.HRegion(8855): Flush requested on be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:38,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. as already flushing 2024-11-11T20:55:38,657 INFO [Thread-2137 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2137. 2024-11-11T20:55:38,657 INFO [Thread-2139 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2139. 2024-11-11T20:55:38,657 DEBUG [Thread-2137 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:55:38,657 DEBUG [Thread-2139 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:55:38,657 DEBUG [Thread-2137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:38,657 DEBUG [Thread-2139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:38,657 DEBUG [Thread-2139 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:38,657 DEBUG [Thread-2137 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:38,657 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:55:38,658 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:55:38,667 INFO [Thread-2133 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2133. 2024-11-11T20:55:38,667 DEBUG [Thread-2133 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:55:38,667 DEBUG [Thread-2133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:38,668 DEBUG [Thread-2133 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:38,668 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:55:38,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:55:38,722 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111eb92f4abb404bdf99423c23f15fbeb8_be56906506ad5c3978d05dd159fda63b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111eb92f4abb404bdf99423c23f15fbeb8_be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:38,724 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/7938deb043554eaa96e19927428da9c5, store: [table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:38,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/7938deb043554eaa96e19927428da9c5 is 175, key is test_row_0/A:col10/1731358536503/Put/seqid=0 2024-11-11T20:55:38,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742365_1541 (size=31105) 2024-11-11T20:55:38,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T20:55:39,000 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/95f69896a8f84deca43bbd50bc66f881 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/95f69896a8f84deca43bbd50bc66f881 2024-11-11T20:55:39,008 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be56906506ad5c3978d05dd159fda63b/A of be56906506ad5c3978d05dd159fda63b into 95f69896a8f84deca43bbd50bc66f881(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:55:39,008 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be56906506ad5c3978d05dd159fda63b: 2024-11-11T20:55:39,008 INFO [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., storeName=be56906506ad5c3978d05dd159fda63b/A, priority=13, startTime=1731358538167; duration=0sec 2024-11-11T20:55:39,008 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:39,008 DEBUG [RS:0;51ca66f7ee3c:44669-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be56906506ad5c3978d05dd159fda63b:A 2024-11-11T20:55:39,011 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/e7fecfb1f7a44592b62cfe852f4e2d45 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/e7fecfb1f7a44592b62cfe852f4e2d45 2024-11-11T20:55:39,015 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in be56906506ad5c3978d05dd159fda63b/C of be56906506ad5c3978d05dd159fda63b into e7fecfb1f7a44592b62cfe852f4e2d45(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T20:55:39,015 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for be56906506ad5c3978d05dd159fda63b: 2024-11-11T20:55:39,015 INFO [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., storeName=be56906506ad5c3978d05dd159fda63b/C, priority=13, startTime=1731358538168; duration=0sec 2024-11-11T20:55:39,015 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T20:55:39,015 DEBUG [RS:0;51ca66f7ee3c:44669-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: be56906506ad5c3978d05dd159fda63b:C 2024-11-11T20:55:39,130 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/7938deb043554eaa96e19927428da9c5 2024-11-11T20:55:39,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/a9e94cae99d649a29175c80a0eaccdc8 is 50, key is test_row_0/B:col10/1731358536503/Put/seqid=0 2024-11-11T20:55:39,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742366_1542 (size=12151) 2024-11-11T20:55:39,548 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/a9e94cae99d649a29175c80a0eaccdc8 2024-11-11T20:55:39,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/23261105a1ce466e90f9d10c4a0eb3cb is 50, key is test_row_0/C:col10/1731358536503/Put/seqid=0 2024-11-11T20:55:39,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742367_1543 (size=12151) 2024-11-11T20:55:39,967 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/23261105a1ce466e90f9d10c4a0eb3cb 2024-11-11T20:55:39,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/7938deb043554eaa96e19927428da9c5 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/7938deb043554eaa96e19927428da9c5 2024-11-11T20:55:39,982 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/7938deb043554eaa96e19927428da9c5, entries=150, sequenceid=224, filesize=30.4 K 2024-11-11T20:55:39,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/a9e94cae99d649a29175c80a0eaccdc8 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/a9e94cae99d649a29175c80a0eaccdc8 2024-11-11T20:55:39,985 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/a9e94cae99d649a29175c80a0eaccdc8, entries=150, sequenceid=224, filesize=11.9 K 2024-11-11T20:55:39,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/23261105a1ce466e90f9d10c4a0eb3cb as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/23261105a1ce466e90f9d10c4a0eb3cb 2024-11-11T20:55:39,988 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/23261105a1ce466e90f9d10c4a0eb3cb, entries=150, sequenceid=224, filesize=11.9 K 2024-11-11T20:55:39,988 INFO [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=20.13 KB/20610 for be56906506ad5c3978d05dd159fda63b in 1689ms, sequenceid=224, compaction requested=false 2024-11-11T20:55:39,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for be56906506ad5c3978d05dd159fda63b: 2024-11-11T20:55:39,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:39,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/51ca66f7ee3c:0-1 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-11T20:55:39,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-11T20:55:39,991 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-11T20:55:39,991 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3800 sec 2024-11-11T20:55:39,992 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 3.3840 sec 2024-11-11T20:55:40,578 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T20:55:40,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-11T20:55:40,777 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:55:45,991 INFO [Thread-2135 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2135. 2024-11-11T20:55:45,991 DEBUG [Thread-2135 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:55:45,991 DEBUG [Thread-2135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:45,991 DEBUG [Thread-2135 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:45,991 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:55:46,018 INFO [Thread-2131 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2131. 2024-11-11T20:55:46,018 DEBUG [Thread-2131 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T20:55:46,018 DEBUG [Thread-2131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:46,018 DEBUG [Thread-2131 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:46,018 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:55:46,018 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T20:55:46,018 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 6 2024-11-11T20:55:46,018 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-11T20:55:46,018 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 7 2024-11-11T20:55:46,018 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 52 2024-11-11T20:55:46,018 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 54 2024-11-11T20:55:46,018 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T20:55:46,018 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3598 2024-11-11T20:55:46,018 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3784 2024-11-11T20:55:46,018 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3617 2024-11-11T20:55:46,018 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3607 2024-11-11T20:55:46,018 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3805 2024-11-11T20:55:46,018 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T20:55:46,018 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T20:55:46,019 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:55:46,019 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:46,019 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:46,019 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:55:46,019 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T20:55:46,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T20:55:46,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T20:55:46,023 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358546023"}]},"ts":"1731358546023"} 2024-11-11T20:55:46,024 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T20:55:46,025 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T20:55:46,025 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T20:55:46,027 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=be56906506ad5c3978d05dd159fda63b, UNASSIGN}] 2024-11-11T20:55:46,028 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=152, ppid=151, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=be56906506ad5c3978d05dd159fda63b, UNASSIGN 2024-11-11T20:55:46,029 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=152 updating hbase:meta row=be56906506ad5c3978d05dd159fda63b, regionState=CLOSING, regionLocation=51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:46,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=152, ppid=151, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=be56906506ad5c3978d05dd159fda63b, UNASSIGN because future has completed 2024-11-11T20:55:46,031 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T20:55:46,031 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; CloseRegionProcedure be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066}] 2024-11-11T20:55:46,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T20:55:46,185 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] handler.UnassignRegionHandler(122): Close be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:46,185 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T20:55:46,185 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1722): Closing be56906506ad5c3978d05dd159fda63b, disabling compactions & flushes 2024-11-11T20:55:46,185 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:46,185 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:46,185 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. after waiting 0 ms 2024-11-11T20:55:46,185 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:46,186 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(2902): Flushing be56906506ad5c3978d05dd159fda63b 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T20:55:46,186 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=A 2024-11-11T20:55:46,186 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:46,186 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=B 2024-11-11T20:55:46,187 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:46,187 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK be56906506ad5c3978d05dd159fda63b, store=C 2024-11-11T20:55:46,187 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T20:55:46,197 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111158b4676698d348fc9f8614eeba0e984c_be56906506ad5c3978d05dd159fda63b is 50, key is test_row_0/A:col10/1731358545987/Put/seqid=0 2024-11-11T20:55:46,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742368_1544 (size=12304) 2024-11-11T20:55:46,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T20:55:46,602 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T20:55:46,611 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111158b4676698d348fc9f8614eeba0e984c_be56906506ad5c3978d05dd159fda63b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111158b4676698d348fc9f8614eeba0e984c_be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:46,612 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/65a72c53f4314f9f8ffcaa9106c6eece, store: [table=TestAcidGuarantees family=A region=be56906506ad5c3978d05dd159fda63b] 2024-11-11T20:55:46,613 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/65a72c53f4314f9f8ffcaa9106c6eece is 175, key is test_row_0/A:col10/1731358545987/Put/seqid=0 2024-11-11T20:55:46,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742369_1545 (size=31105) 2024-11-11T20:55:46,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T20:55:47,020 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/65a72c53f4314f9f8ffcaa9106c6eece 2024-11-11T20:55:47,031 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/e11b75e5e6be4a3cb6358b7399572b4a is 50, key is test_row_0/B:col10/1731358545987/Put/seqid=0 2024-11-11T20:55:47,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742370_1546 (size=12151) 2024-11-11T20:55:47,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T20:55:47,436 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/e11b75e5e6be4a3cb6358b7399572b4a 2024-11-11T20:55:47,449 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/0fbfa0fd2bc442b9aa6ec9647ad4ebdc is 50, key is test_row_0/C:col10/1731358545987/Put/seqid=0 2024-11-11T20:55:47,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742371_1547 (size=12151) 2024-11-11T20:55:47,855 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/0fbfa0fd2bc442b9aa6ec9647ad4ebdc 2024-11-11T20:55:47,865 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/A/65a72c53f4314f9f8ffcaa9106c6eece as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/65a72c53f4314f9f8ffcaa9106c6eece 2024-11-11T20:55:47,869 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/65a72c53f4314f9f8ffcaa9106c6eece, entries=150, sequenceid=235, filesize=30.4 K 2024-11-11T20:55:47,870 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/B/e11b75e5e6be4a3cb6358b7399572b4a as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/e11b75e5e6be4a3cb6358b7399572b4a 2024-11-11T20:55:47,874 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/e11b75e5e6be4a3cb6358b7399572b4a, entries=150, sequenceid=235, filesize=11.9 K 2024-11-11T20:55:47,875 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/.tmp/C/0fbfa0fd2bc442b9aa6ec9647ad4ebdc as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/0fbfa0fd2bc442b9aa6ec9647ad4ebdc 2024-11-11T20:55:47,879 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/0fbfa0fd2bc442b9aa6ec9647ad4ebdc, entries=150, sequenceid=235, filesize=11.9 K 2024-11-11T20:55:47,880 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for be56906506ad5c3978d05dd159fda63b in 1695ms, sequenceid=235, compaction requested=true 2024-11-11T20:55:47,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/be0f3c0549a5467b8a96219e05f67df3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/f3cc5f25da694622ae5a591ab5817fdf, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/4daae7d3fb02488ea432ac24947275ab, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/5ffeface8cdc42b79063e9acc4baa192, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/aa8705187b844618a676e226d3024dd0, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/4ae607f23dfb4ac9b79dea0f193e4e19, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/05fdc3c97d6e46b8ac605e7001012451, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/d596b833e82541cca9b4d06cbe719152, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/2a6658ed6ec94b5c9e06e51520ea997d, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/13da9719668b46dc9374f20742f7eff7, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/a690425599d944e9a93f90a162ab7e67, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/6e5b69024248455abfc3d96287fbdc7c, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/ce22d93a65ec4fd7b860d8c57a62c334, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/5e643a6043874604b93e807fce833174, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/858f956f7b344815b4e96d7bb96bf890, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/25504385f41442a598985c29de1bb566] to archive 2024-11-11T20:55:47,882 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:55:47,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/be0f3c0549a5467b8a96219e05f67df3 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/be0f3c0549a5467b8a96219e05f67df3 2024-11-11T20:55:47,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/f3cc5f25da694622ae5a591ab5817fdf to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/f3cc5f25da694622ae5a591ab5817fdf 2024-11-11T20:55:47,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/4daae7d3fb02488ea432ac24947275ab to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/4daae7d3fb02488ea432ac24947275ab 2024-11-11T20:55:47,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/5ffeface8cdc42b79063e9acc4baa192 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/5ffeface8cdc42b79063e9acc4baa192 2024-11-11T20:55:47,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/aa8705187b844618a676e226d3024dd0 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/aa8705187b844618a676e226d3024dd0 2024-11-11T20:55:47,890 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/4ae607f23dfb4ac9b79dea0f193e4e19 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/4ae607f23dfb4ac9b79dea0f193e4e19 2024-11-11T20:55:47,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/05fdc3c97d6e46b8ac605e7001012451 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/05fdc3c97d6e46b8ac605e7001012451 2024-11-11T20:55:47,892 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/d596b833e82541cca9b4d06cbe719152 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/d596b833e82541cca9b4d06cbe719152 2024-11-11T20:55:47,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/2a6658ed6ec94b5c9e06e51520ea997d to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/2a6658ed6ec94b5c9e06e51520ea997d 2024-11-11T20:55:47,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/13da9719668b46dc9374f20742f7eff7 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/13da9719668b46dc9374f20742f7eff7 2024-11-11T20:55:47,896 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/a690425599d944e9a93f90a162ab7e67 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/a690425599d944e9a93f90a162ab7e67 2024-11-11T20:55:47,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/6e5b69024248455abfc3d96287fbdc7c to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/6e5b69024248455abfc3d96287fbdc7c 2024-11-11T20:55:47,898 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/ce22d93a65ec4fd7b860d8c57a62c334 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/ce22d93a65ec4fd7b860d8c57a62c334 2024-11-11T20:55:47,900 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/5e643a6043874604b93e807fce833174 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/5e643a6043874604b93e807fce833174 2024-11-11T20:55:47,901 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/858f956f7b344815b4e96d7bb96bf890 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/858f956f7b344815b4e96d7bb96bf890 2024-11-11T20:55:47,903 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/25504385f41442a598985c29de1bb566 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/25504385f41442a598985c29de1bb566 2024-11-11T20:55:47,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/93a8dff664da432fbf3eaff99d61b713, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/c33a331605d04ee8be1ada102029750a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/427230f421a04a289e333694e330f59a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/cdf97d63202b47909409a5df0fa072db, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/bfc10be5d45947d886f1ec252b57460b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/54e9eb0337b4414cb5a6192394f8794d, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/2c24de5a8c364aa196aa51258b0523a3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/bf971f5dd91645eca4045d02ed96a8d9, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/daf3b3d8437249e098e051c78ef9bd93, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/0f44b67a11d2414ba6d9f91d8644104a, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/6c8028d7d1ea4c0c9733e64b65ea74d3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/bf7da0749f90472cb96e5104162f36ed, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/871e48d60e35440095992c6cdae2c202, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/6e9a481cdea34b5980403885c2c5e711, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/e4189b68d07348bead8d965c8a87ae8d, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/ebd22342e8944a8eb79343f59c678af9] to archive 2024-11-11T20:55:47,906 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:55:47,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/93a8dff664da432fbf3eaff99d61b713 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/93a8dff664da432fbf3eaff99d61b713 2024-11-11T20:55:47,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/c33a331605d04ee8be1ada102029750a to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/c33a331605d04ee8be1ada102029750a 2024-11-11T20:55:47,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/427230f421a04a289e333694e330f59a to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/427230f421a04a289e333694e330f59a 2024-11-11T20:55:47,910 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/cdf97d63202b47909409a5df0fa072db to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/cdf97d63202b47909409a5df0fa072db 2024-11-11T20:55:47,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/bfc10be5d45947d886f1ec252b57460b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/bfc10be5d45947d886f1ec252b57460b 2024-11-11T20:55:47,912 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/54e9eb0337b4414cb5a6192394f8794d to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/54e9eb0337b4414cb5a6192394f8794d 2024-11-11T20:55:47,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/2c24de5a8c364aa196aa51258b0523a3 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/2c24de5a8c364aa196aa51258b0523a3 2024-11-11T20:55:47,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/bf971f5dd91645eca4045d02ed96a8d9 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/bf971f5dd91645eca4045d02ed96a8d9 2024-11-11T20:55:47,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/daf3b3d8437249e098e051c78ef9bd93 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/daf3b3d8437249e098e051c78ef9bd93 2024-11-11T20:55:47,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/0f44b67a11d2414ba6d9f91d8644104a to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/0f44b67a11d2414ba6d9f91d8644104a 2024-11-11T20:55:47,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/6c8028d7d1ea4c0c9733e64b65ea74d3 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/6c8028d7d1ea4c0c9733e64b65ea74d3 2024-11-11T20:55:47,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/bf7da0749f90472cb96e5104162f36ed to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/bf7da0749f90472cb96e5104162f36ed 2024-11-11T20:55:47,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/871e48d60e35440095992c6cdae2c202 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/871e48d60e35440095992c6cdae2c202 2024-11-11T20:55:47,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/6e9a481cdea34b5980403885c2c5e711 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/6e9a481cdea34b5980403885c2c5e711 2024-11-11T20:55:47,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/e4189b68d07348bead8d965c8a87ae8d to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/e4189b68d07348bead8d965c8a87ae8d 2024-11-11T20:55:47,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/ebd22342e8944a8eb79343f59c678af9 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/ebd22342e8944a8eb79343f59c678af9 2024-11-11T20:55:47,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/1627b74bdf0f47bb9752764d4b9bdf75, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/437af3df1ac14d6a86cc3b83f493b133, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/b2e1282b8c4946d3bb6628e2fa505ad1, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/4049bd6d5945447592a28376fcea8184, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/5e16603363664108a637e153d68b1cb6, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/1fa941b936924ed58addf5524a26b5f5, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/4fe91cc4e4f349fbbd468539325fee9d, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/a9cab78c72af4540b644d29506c4ebe3, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/294df39a445d47c88efd3d99269f4b4b, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/d1398877a709405d8b7922a93c56dcd0, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/5833fbae9f34433aa351ff512200f55d, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/af8b839743bf448b93c668c91fa0b1f4, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/97bf0fa267d84efd848c534ae6eadce2, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/c085d32529bd4af096b8dd60efd0b524, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/25579a69e8db4e2c9f397668b529aec8, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/e9463852ecaa4150936733e339613637] to archive 2024-11-11T20:55:47,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T20:55:47,921 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/1627b74bdf0f47bb9752764d4b9bdf75 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/1627b74bdf0f47bb9752764d4b9bdf75 2024-11-11T20:55:47,922 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/437af3df1ac14d6a86cc3b83f493b133 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/437af3df1ac14d6a86cc3b83f493b133 2024-11-11T20:55:47,923 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/b2e1282b8c4946d3bb6628e2fa505ad1 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/b2e1282b8c4946d3bb6628e2fa505ad1 2024-11-11T20:55:47,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/4049bd6d5945447592a28376fcea8184 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/4049bd6d5945447592a28376fcea8184 2024-11-11T20:55:47,925 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/5e16603363664108a637e153d68b1cb6 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/5e16603363664108a637e153d68b1cb6 2024-11-11T20:55:47,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/1fa941b936924ed58addf5524a26b5f5 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/1fa941b936924ed58addf5524a26b5f5 2024-11-11T20:55:47,926 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/4fe91cc4e4f349fbbd468539325fee9d to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/4fe91cc4e4f349fbbd468539325fee9d 2024-11-11T20:55:47,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/a9cab78c72af4540b644d29506c4ebe3 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/a9cab78c72af4540b644d29506c4ebe3 2024-11-11T20:55:47,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/294df39a445d47c88efd3d99269f4b4b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/294df39a445d47c88efd3d99269f4b4b 2024-11-11T20:55:47,928 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/d1398877a709405d8b7922a93c56dcd0 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/d1398877a709405d8b7922a93c56dcd0 2024-11-11T20:55:47,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/5833fbae9f34433aa351ff512200f55d to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/5833fbae9f34433aa351ff512200f55d 2024-11-11T20:55:47,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/af8b839743bf448b93c668c91fa0b1f4 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/af8b839743bf448b93c668c91fa0b1f4 2024-11-11T20:55:47,930 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/97bf0fa267d84efd848c534ae6eadce2 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/97bf0fa267d84efd848c534ae6eadce2 2024-11-11T20:55:47,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/c085d32529bd4af096b8dd60efd0b524 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/c085d32529bd4af096b8dd60efd0b524 2024-11-11T20:55:47,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/25579a69e8db4e2c9f397668b529aec8 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/25579a69e8db4e2c9f397668b529aec8 2024-11-11T20:55:47,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/e9463852ecaa4150936733e339613637 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/e9463852ecaa4150936733e339613637 2024-11-11T20:55:47,936 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/recovered.edits/238.seqid, newMaxSeqId=238, maxSeqId=4 2024-11-11T20:55:47,936 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. 2024-11-11T20:55:47,936 DEBUG [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] regionserver.HRegion(1676): Region close journal for be56906506ad5c3978d05dd159fda63b: Waiting for close lock at 1731358546185Running coprocessor pre-close hooks at 1731358546185Disabling compacts and flushes for region at 1731358546185Disabling writes for close at 1731358546185Obtaining lock to block concurrent updates at 1731358546186 (+1 ms)Preparing flush snapshotting stores in be56906506ad5c3978d05dd159fda63b at 1731358546186Finished memstore snapshotting TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731358546187 (+1 ms)Flushing stores of TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b. at 1731358546188 (+1 ms)Flushing be56906506ad5c3978d05dd159fda63b/A: creating writer at 1731358546188Flushing mob file be56906506ad5c3978d05dd159fda63b/A: appending metadata at 1731358546197 (+9 ms)Flushing mob file be56906506ad5c3978d05dd159fda63b/A: closing flushed file at 1731358546197Flushing be56906506ad5c3978d05dd159fda63b/A: appending metadata at 1731358546612 (+415 ms)Flushing be56906506ad5c3978d05dd159fda63b/A: closing flushed file at 1731358546613 (+1 ms)Flushing be56906506ad5c3978d05dd159fda63b/B: creating writer at 1731358547029 (+416 ms)Flushing be56906506ad5c3978d05dd159fda63b/B: appending metadata at 1731358547031 (+2 ms)Flushing be56906506ad5c3978d05dd159fda63b/B: closing flushed file at 1731358547031Flushing be56906506ad5c3978d05dd159fda63b/C: creating writer at 1731358547446 (+415 ms)Flushing be56906506ad5c3978d05dd159fda63b/C: appending metadata at 1731358547449 (+3 ms)Flushing be56906506ad5c3978d05dd159fda63b/C: closing flushed file at 1731358547449Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@162e1803: reopening flushed file at 1731358547864 (+415 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5657376b: reopening flushed file at 1731358547869 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33b364b7: reopening flushed file at 1731358547874 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for be56906506ad5c3978d05dd159fda63b in 1695ms, sequenceid=235, compaction requested=true at 1731358547880 (+6 ms)Writing region close event to WAL at 1731358547933 (+53 ms)Running coprocessor post-close hooks at 1731358547936 (+3 ms)Closed at 1731358547936 2024-11-11T20:55:47,938 INFO [RS_CLOSE_REGION-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_REGION, pid=153}] handler.UnassignRegionHandler(157): Closed be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:47,938 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=152 updating hbase:meta row=be56906506ad5c3978d05dd159fda63b, regionState=CLOSED 2024-11-11T20:55:47,939 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE, hasLock=false; CloseRegionProcedure be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 because future has completed 2024-11-11T20:55:47,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-11T20:55:47,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; CloseRegionProcedure be56906506ad5c3978d05dd159fda63b, server=51ca66f7ee3c,44669,1731358362066 in 1.9090 sec 2024-11-11T20:55:47,942 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-11-11T20:55:47,942 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=be56906506ad5c3978d05dd159fda63b, UNASSIGN in 1.9140 sec 2024-11-11T20:55:47,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-11T20:55:47,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9180 sec 2024-11-11T20:55:47,945 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731358547945"}]},"ts":"1731358547945"} 2024-11-11T20:55:47,946 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T20:55:47,946 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T20:55:47,948 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9270 sec 2024-11-11T20:55:48,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-11T20:55:48,167 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:55:48,168 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T20:55:48,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:55:48,172 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=154, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:55:48,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T20:55:48,174 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=154, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:55:48,177 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:48,179 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A, FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B, FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C, FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/recovered.edits] 2024-11-11T20:55:48,181 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/65a72c53f4314f9f8ffcaa9106c6eece to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/65a72c53f4314f9f8ffcaa9106c6eece 2024-11-11T20:55:48,182 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/7938deb043554eaa96e19927428da9c5 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/7938deb043554eaa96e19927428da9c5 2024-11-11T20:55:48,184 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/95f69896a8f84deca43bbd50bc66f881 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/A/95f69896a8f84deca43bbd50bc66f881 2024-11-11T20:55:48,186 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/12f359bda0bc4484981d522bb0364f1e to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/12f359bda0bc4484981d522bb0364f1e 2024-11-11T20:55:48,187 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/a9e94cae99d649a29175c80a0eaccdc8 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/a9e94cae99d649a29175c80a0eaccdc8 2024-11-11T20:55:48,188 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/e11b75e5e6be4a3cb6358b7399572b4a to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/B/e11b75e5e6be4a3cb6358b7399572b4a 2024-11-11T20:55:48,190 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/0fbfa0fd2bc442b9aa6ec9647ad4ebdc to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/0fbfa0fd2bc442b9aa6ec9647ad4ebdc 2024-11-11T20:55:48,192 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/23261105a1ce466e90f9d10c4a0eb3cb to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/23261105a1ce466e90f9d10c4a0eb3cb 2024-11-11T20:55:48,193 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/e7fecfb1f7a44592b62cfe852f4e2d45 to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/C/e7fecfb1f7a44592b62cfe852f4e2d45 2024-11-11T20:55:48,196 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/recovered.edits/238.seqid to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b/recovered.edits/238.seqid 2024-11-11T20:55:48,197 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/default/TestAcidGuarantees/be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:48,197 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T20:55:48,197 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T20:55:48,198 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T20:55:48,201 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111b88d966285b447399da7445525713fe_be56906506ad5c3978d05dd159fda63b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111b88d966285b447399da7445525713fe_be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:48,203 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111eb92f4abb404bdf99423c23f15fbeb8_be56906506ad5c3978d05dd159fda63b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111eb92f4abb404bdf99423c23f15fbeb8_be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:48,204 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111123288af8ee3e422e99dd5e749c8e0a51_be56906506ad5c3978d05dd159fda63b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111123288af8ee3e422e99dd5e749c8e0a51_be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:48,207 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112a161b804c3f419dbe9571bae9c80f36_be56906506ad5c3978d05dd159fda63b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112a161b804c3f419dbe9571bae9c80f36_be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:48,209 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111158b4676698d348fc9f8614eeba0e984c_be56906506ad5c3978d05dd159fda63b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111158b4676698d348fc9f8614eeba0e984c_be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:48,211 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111161503d67288e49fda9f798738b1e5f2d_be56906506ad5c3978d05dd159fda63b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111161503d67288e49fda9f798738b1e5f2d_be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:48,212 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111169603aaba4ca45a493f5c93d23e5909c_be56906506ad5c3978d05dd159fda63b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111169603aaba4ca45a493f5c93d23e5909c_be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:48,214 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111174fa18d388a749dea32ea2e4491d4e38_be56906506ad5c3978d05dd159fda63b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111174fa18d388a749dea32ea2e4491d4e38_be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:48,215 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118be64410a2e841e29a2f4fb6c6e9fb57_be56906506ad5c3978d05dd159fda63b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118be64410a2e841e29a2f4fb6c6e9fb57_be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:48,217 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b01ca1ba1fb2461dae657c884448e2b9_be56906506ad5c3978d05dd159fda63b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b01ca1ba1fb2461dae657c884448e2b9_be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:48,218 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b98abb3615b0468d902d951a05d7bbd9_be56906506ad5c3978d05dd159fda63b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b98abb3615b0468d902d951a05d7bbd9_be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:48,220 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ccbbe8a4c55f4d10a4bfd74fe429c630_be56906506ad5c3978d05dd159fda63b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ccbbe8a4c55f4d10a4bfd74fe429c630_be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:48,221 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ce4fc5bef02b400c82d91bbc16b0f831_be56906506ad5c3978d05dd159fda63b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ce4fc5bef02b400c82d91bbc16b0f831_be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:48,223 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d39d1a7d20634ac983b2350e250b6af8_be56906506ad5c3978d05dd159fda63b to hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d39d1a7d20634ac983b2350e250b6af8_be56906506ad5c3978d05dd159fda63b 2024-11-11T20:55:48,224 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T20:55:48,228 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=154, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:55:48,231 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T20:55:48,234 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T20:55:48,235 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=154, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:55:48,235 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T20:55:48,235 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731358548235"}]},"ts":"9223372036854775807"} 2024-11-11T20:55:48,238 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T20:55:48,238 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => be56906506ad5c3978d05dd159fda63b, NAME => 'TestAcidGuarantees,,1731358514500.be56906506ad5c3978d05dd159fda63b.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T20:55:48,238 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T20:55:48,238 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731358548238"}]},"ts":"9223372036854775807"} 2024-11-11T20:55:48,240 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T20:55:48,240 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=154, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T20:55:48,242 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 72 msec 2024-11-11T20:55:48,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38429 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-11T20:55:48,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T20:55:48,279 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T20:55:48,291 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=247 (was 249), OpenFileDescriptor=445 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=274 (was 286), ProcessCount=11 (was 11), AvailableMemoryMB=3996 (was 3998) 2024-11-11T20:55:48,291 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-11T20:55:48,291 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T20:55:48,292 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:55:48,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:48,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:48,292 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T20:55:48,292 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-11T20:55:48,292 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1107072306, stopped=false 2024-11-11T20:55:48,293 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=51ca66f7ee3c,38429,1731358361462 2024-11-11T20:55:48,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44669-0x100308f60b00001, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T20:55:48,294 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T20:55:48,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:55:48,295 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44669-0x100308f60b00001, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:55:48,295 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:55:48,295 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T20:55:48,295 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:55:48,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:48,295 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44669-0x100308f60b00001, quorum=127.0.0.1:61333, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T20:55:48,295 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T20:55:48,296 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '51ca66f7ee3c,44669,1731358362066' ***** 2024-11-11T20:55:48,296 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T20:55:48,296 INFO [RS:0;51ca66f7ee3c:44669 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T20:55:48,296 INFO [RS:0;51ca66f7ee3c:44669 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T20:55:48,296 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T20:55:48,296 INFO [RS:0;51ca66f7ee3c:44669 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T20:55:48,296 INFO [RS:0;51ca66f7ee3c:44669 {}] regionserver.HRegionServer(959): stopping server 51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:48,296 INFO [RS:0;51ca66f7ee3c:44669 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T20:55:48,296 INFO [RS:0;51ca66f7ee3c:44669 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;51ca66f7ee3c:44669. 2024-11-11T20:55:48,297 DEBUG [RS:0;51ca66f7ee3c:44669 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T20:55:48,297 DEBUG [RS:0;51ca66f7ee3c:44669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:48,297 INFO [RS:0;51ca66f7ee3c:44669 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T20:55:48,297 INFO [RS:0;51ca66f7ee3c:44669 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T20:55:48,297 INFO [RS:0;51ca66f7ee3c:44669 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T20:55:48,297 INFO [RS:0;51ca66f7ee3c:44669 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-11T20:55:48,297 INFO [RS:0;51ca66f7ee3c:44669 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-11T20:55:48,297 DEBUG [RS:0;51ca66f7ee3c:44669 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-11T20:55:48,298 DEBUG [RS:0;51ca66f7ee3c:44669 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T20:55:48,298 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T20:55:48,298 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T20:55:48,298 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T20:55:48,298 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T20:55:48,298 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T20:55:48,298 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-11T20:55:48,318 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/hbase/meta/1588230740/.tmp/info/ed66159b63dc44cbbc4d24d099bc3bd2 is 95, key is TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044./info:/1731358392747/DeleteFamily/seqid=0 2024-11-11T20:55:48,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742372_1548 (size=6514) 2024-11-11T20:55:48,346 INFO [regionserver/51ca66f7ee3c:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T20:55:48,498 DEBUG [RS:0;51ca66f7ee3c:44669 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T20:55:48,698 DEBUG [RS:0;51ca66f7ee3c:44669 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T20:55:48,723 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/hbase/meta/1588230740/.tmp/info/ed66159b63dc44cbbc4d24d099bc3bd2 2024-11-11T20:55:48,747 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/hbase/meta/1588230740/.tmp/ns/9dc52090780843c9af21a60f96ac21c4 is 93, key is TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044./ns:/1731358392698/DeleteFamily/seqid=0 2024-11-11T20:55:48,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742373_1549 (size=5906) 2024-11-11T20:55:48,898 DEBUG [RS:0;51ca66f7ee3c:44669 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T20:55:48,929 INFO [regionserver/51ca66f7ee3c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-11T20:55:48,930 INFO [regionserver/51ca66f7ee3c:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-11T20:55:49,099 DEBUG [RS:0;51ca66f7ee3c:44669 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T20:55:49,151 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/hbase/meta/1588230740/.tmp/ns/9dc52090780843c9af21a60f96ac21c4 2024-11-11T20:55:49,177 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/hbase/meta/1588230740/.tmp/rep_barrier/0c720b9e34534f10845a6abf085159da is 102, key is TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044./rep_barrier:/1731358392698/DeleteFamily/seqid=0 2024-11-11T20:55:49,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742374_1550 (size=6025) 2024-11-11T20:55:49,299 DEBUG [RS:0;51ca66f7ee3c:44669 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T20:55:49,499 DEBUG [RS:0;51ca66f7ee3c:44669 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T20:55:49,581 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/hbase/meta/1588230740/.tmp/rep_barrier/0c720b9e34534f10845a6abf085159da 2024-11-11T20:55:49,606 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/hbase/meta/1588230740/.tmp/table/716cf7d794b94a2193e9c05ea15abc04 is 96, key is TestAcidGuarantees,,1731358366090.09438051ac0c3c1f634095c776d91044./table:/1731358392698/DeleteFamily/seqid=0 2024-11-11T20:55:49,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742375_1551 (size=5936) 2024-11-11T20:55:49,700 DEBUG [RS:0;51ca66f7ee3c:44669 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T20:55:49,900 DEBUG [RS:0;51ca66f7ee3c:44669 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T20:55:50,011 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/hbase/meta/1588230740/.tmp/table/716cf7d794b94a2193e9c05ea15abc04 2024-11-11T20:55:50,021 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/hbase/meta/1588230740/.tmp/info/ed66159b63dc44cbbc4d24d099bc3bd2 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/hbase/meta/1588230740/info/ed66159b63dc44cbbc4d24d099bc3bd2 2024-11-11T20:55:50,027 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/hbase/meta/1588230740/info/ed66159b63dc44cbbc4d24d099bc3bd2, entries=12, sequenceid=90, filesize=6.4 K 2024-11-11T20:55:50,029 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/hbase/meta/1588230740/.tmp/ns/9dc52090780843c9af21a60f96ac21c4 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/hbase/meta/1588230740/ns/9dc52090780843c9af21a60f96ac21c4 2024-11-11T20:55:50,034 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/hbase/meta/1588230740/ns/9dc52090780843c9af21a60f96ac21c4, entries=8, sequenceid=90, filesize=5.8 K 2024-11-11T20:55:50,035 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/hbase/meta/1588230740/.tmp/rep_barrier/0c720b9e34534f10845a6abf085159da as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/hbase/meta/1588230740/rep_barrier/0c720b9e34534f10845a6abf085159da 2024-11-11T20:55:50,039 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/hbase/meta/1588230740/rep_barrier/0c720b9e34534f10845a6abf085159da, entries=6, sequenceid=90, filesize=5.9 K 2024-11-11T20:55:50,040 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/hbase/meta/1588230740/.tmp/table/716cf7d794b94a2193e9c05ea15abc04 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/hbase/meta/1588230740/table/716cf7d794b94a2193e9c05ea15abc04 2024-11-11T20:55:50,043 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/hbase/meta/1588230740/table/716cf7d794b94a2193e9c05ea15abc04, entries=7, sequenceid=90, filesize=5.8 K 2024-11-11T20:55:50,044 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1746ms, sequenceid=90, compaction requested=false 2024-11-11T20:55:50,050 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-11T20:55:50,050 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T20:55:50,051 INFO [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T20:55:50,051 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731358548297Running coprocessor pre-close hooks at 1731358548298 (+1 ms)Disabling compacts and flushes for region at 1731358548298Disabling writes for close at 1731358548298Obtaining lock to block concurrent updates at 1731358548298Preparing flush snapshotting stores in 1588230740 at 1731358548298Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731358548298Flushing stores of hbase:meta,,1.1588230740 at 1731358548299 (+1 ms)Flushing 1588230740/info: creating writer at 1731358548299Flushing 1588230740/info: appending metadata at 1731358548318 (+19 ms)Flushing 1588230740/info: closing flushed file at 1731358548318Flushing 1588230740/ns: creating writer at 1731358548731 (+413 ms)Flushing 1588230740/ns: appending metadata at 1731358548746 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1731358548746Flushing 1588230740/rep_barrier: creating writer at 1731358549161 (+415 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731358549177 (+16 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731358549177Flushing 1588230740/table: creating writer at 1731358549586 (+409 ms)Flushing 1588230740/table: appending metadata at 1731358549606 (+20 ms)Flushing 1588230740/table: closing flushed file at 1731358549606Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56cd0917: reopening flushed file at 1731358550019 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f15f1bb: reopening flushed file at 1731358550028 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34a957d6: reopening flushed file at 1731358550034 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e2cf535: reopening flushed file at 1731358550039 (+5 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1746ms, sequenceid=90, compaction requested=false at 1731358550044 (+5 ms)Writing region close event to WAL at 1731358550046 (+2 ms)Running coprocessor post-close hooks at 1731358550050 (+4 ms)Closed at 1731358550050 2024-11-11T20:55:50,051 DEBUG [RS_CLOSE_META-regionserver/51ca66f7ee3c:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-11T20:55:50,100 INFO [RS:0;51ca66f7ee3c:44669 {}] regionserver.HRegionServer(976): stopping server 51ca66f7ee3c,44669,1731358362066; all regions closed. 2024-11-11T20:55:50,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741834_1010 (size=25344) 2024-11-11T20:55:50,110 DEBUG [RS:0;51ca66f7ee3c:44669 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/oldWALs 2024-11-11T20:55:50,111 INFO [RS:0;51ca66f7ee3c:44669 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 51ca66f7ee3c%2C44669%2C1731358362066.meta:.meta(num 1731358365676) 2024-11-11T20:55:50,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741833_1009 (size=17905187) 2024-11-11T20:55:50,116 DEBUG [RS:0;51ca66f7ee3c:44669 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/oldWALs 2024-11-11T20:55:50,116 INFO [RS:0;51ca66f7ee3c:44669 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 51ca66f7ee3c%2C44669%2C1731358362066:(num 1731358365099) 2024-11-11T20:55:50,116 DEBUG [RS:0;51ca66f7ee3c:44669 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T20:55:50,116 INFO [RS:0;51ca66f7ee3c:44669 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T20:55:50,116 INFO [RS:0;51ca66f7ee3c:44669 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T20:55:50,116 INFO [RS:0;51ca66f7ee3c:44669 {}] hbase.ChoreService(370): Chore service for: regionserver/51ca66f7ee3c:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-11T20:55:50,116 INFO [RS:0;51ca66f7ee3c:44669 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T20:55:50,117 INFO [regionserver/51ca66f7ee3c:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T20:55:50,118 INFO [RS:0;51ca66f7ee3c:44669 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:44669 2024-11-11T20:55:50,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44669-0x100308f60b00001, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/51ca66f7ee3c,44669,1731358362066 2024-11-11T20:55:50,120 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T20:55:50,120 INFO [RS:0;51ca66f7ee3c:44669 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T20:55:50,121 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [51ca66f7ee3c,44669,1731358362066] 2024-11-11T20:55:50,122 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/51ca66f7ee3c,44669,1731358362066 already deleted, retry=false 2024-11-11T20:55:50,122 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 51ca66f7ee3c,44669,1731358362066 expired; onlineServers=0 2024-11-11T20:55:50,122 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '51ca66f7ee3c,38429,1731358361462' ***** 2024-11-11T20:55:50,122 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-11T20:55:50,122 INFO [M:0;51ca66f7ee3c:38429 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T20:55:50,123 INFO [M:0;51ca66f7ee3c:38429 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T20:55:50,123 DEBUG [M:0;51ca66f7ee3c:38429 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-11T20:55:50,123 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-11T20:55:50,123 DEBUG [M:0;51ca66f7ee3c:38429 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-11T20:55:50,123 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.small.0-1731358364797 {}] cleaner.HFileCleaner(306): Exit Thread[master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.small.0-1731358364797,5,FailOnTimeoutGroup] 2024-11-11T20:55:50,123 DEBUG [master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.large.0-1731358364794 {}] cleaner.HFileCleaner(306): Exit Thread[master/51ca66f7ee3c:0:becomeActiveMaster-HFileCleaner.large.0-1731358364794,5,FailOnTimeoutGroup] 2024-11-11T20:55:50,123 INFO [M:0;51ca66f7ee3c:38429 {}] hbase.ChoreService(370): Chore service for: master/51ca66f7ee3c:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-11T20:55:50,123 INFO [M:0;51ca66f7ee3c:38429 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T20:55:50,123 DEBUG [M:0;51ca66f7ee3c:38429 {}] master.HMaster(1795): Stopping service threads 2024-11-11T20:55:50,123 INFO [M:0;51ca66f7ee3c:38429 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-11T20:55:50,123 INFO [M:0;51ca66f7ee3c:38429 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T20:55:50,124 ERROR [M:0;51ca66f7ee3c:38429 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-11T20:55:50,124 INFO [M:0;51ca66f7ee3c:38429 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T20:55:50,124 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-11T20:55:50,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-11T20:55:50,124 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T20:55:50,125 DEBUG [M:0;51ca66f7ee3c:38429 {}] zookeeper.ZKUtil(347): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-11T20:55:50,125 WARN [M:0;51ca66f7ee3c:38429 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-11T20:55:50,125 INFO [M:0;51ca66f7ee3c:38429 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/.lastflushedseqids 2024-11-11T20:55:50,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742376_1552 (size=72) 2024-11-11T20:55:50,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44669-0x100308f60b00001, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:55:50,222 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44669-0x100308f60b00001, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:55:50,223 INFO [RS:0;51ca66f7ee3c:44669 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T20:55:50,223 INFO [RS:0;51ca66f7ee3c:44669 {}] regionserver.HRegionServer(1031): Exiting; stopping=51ca66f7ee3c,44669,1731358362066; zookeeper connection closed. 2024-11-11T20:55:50,223 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1fa19d22 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1fa19d22 2024-11-11T20:55:50,224 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-11T20:55:50,537 INFO [M:0;51ca66f7ee3c:38429 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T20:55:50,538 INFO [M:0;51ca66f7ee3c:38429 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-11T20:55:50,538 DEBUG [M:0;51ca66f7ee3c:38429 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T20:55:50,538 INFO [M:0;51ca66f7ee3c:38429 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:55:50,538 DEBUG [M:0;51ca66f7ee3c:38429 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:55:50,538 DEBUG [M:0;51ca66f7ee3c:38429 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T20:55:50,539 DEBUG [M:0;51ca66f7ee3c:38429 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:55:50,539 INFO [M:0;51ca66f7ee3c:38429 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=762.26 KB heapSize=933.63 KB 2024-11-11T20:55:50,561 DEBUG [M:0;51ca66f7ee3c:38429 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ffadd5b4ff924798a27321065573b069 is 82, key is hbase:meta,,1/info:regioninfo/1731358365766/Put/seqid=0 2024-11-11T20:55:50,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742377_1553 (size=5672) 2024-11-11T20:55:50,966 INFO [M:0;51ca66f7ee3c:38429 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2125 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ffadd5b4ff924798a27321065573b069 2024-11-11T20:55:50,996 DEBUG [M:0;51ca66f7ee3c:38429 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6a499741a08c45e6abbb3325cfe479fe is 2479, key is \x00\x00\x00\x00\x00\x00\x00\x83/proc:d/1731358517612/Put/seqid=0 2024-11-11T20:55:50,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742378_1554 (size=42567) 2024-11-11T20:55:51,401 INFO [M:0;51ca66f7ee3c:38429 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=761.71 KB at sequenceid=2125 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6a499741a08c45e6abbb3325cfe479fe 2024-11-11T20:55:51,409 INFO [M:0;51ca66f7ee3c:38429 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6a499741a08c45e6abbb3325cfe479fe 2024-11-11T20:55:51,432 DEBUG [M:0;51ca66f7ee3c:38429 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/704d878dfbb7405dba0a75ae6794bef1 is 69, key is 51ca66f7ee3c,44669,1731358362066/rs:state/1731358364857/Put/seqid=0 2024-11-11T20:55:51,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073742379_1555 (size=5156) 2024-11-11T20:55:51,799 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T20:55:51,800 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T20:55:51,801 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T20:55:51,837 INFO [M:0;51ca66f7ee3c:38429 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2125 (bloomFilter=true), to=hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/704d878dfbb7405dba0a75ae6794bef1 2024-11-11T20:55:51,845 DEBUG [M:0;51ca66f7ee3c:38429 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ffadd5b4ff924798a27321065573b069 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ffadd5b4ff924798a27321065573b069 2024-11-11T20:55:51,849 INFO [M:0;51ca66f7ee3c:38429 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ffadd5b4ff924798a27321065573b069, entries=8, sequenceid=2125, filesize=5.5 K 2024-11-11T20:55:51,849 DEBUG [M:0;51ca66f7ee3c:38429 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/6a499741a08c45e6abbb3325cfe479fe as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6a499741a08c45e6abbb3325cfe479fe 2024-11-11T20:55:51,852 INFO [M:0;51ca66f7ee3c:38429 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6a499741a08c45e6abbb3325cfe479fe 2024-11-11T20:55:51,852 INFO [M:0;51ca66f7ee3c:38429 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/6a499741a08c45e6abbb3325cfe479fe, entries=154, sequenceid=2125, filesize=41.6 K 2024-11-11T20:55:51,853 DEBUG [M:0;51ca66f7ee3c:38429 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/704d878dfbb7405dba0a75ae6794bef1 as hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/704d878dfbb7405dba0a75ae6794bef1 2024-11-11T20:55:51,855 INFO [M:0;51ca66f7ee3c:38429 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:45199/user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/704d878dfbb7405dba0a75ae6794bef1, entries=1, sequenceid=2125, filesize=5.0 K 2024-11-11T20:55:51,856 INFO [M:0;51ca66f7ee3c:38429 {}] regionserver.HRegion(3140): Finished flush of dataSize ~762.26 KB/780559, heapSize ~933.33 KB/955728, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1317ms, sequenceid=2125, compaction requested=false 2024-11-11T20:55:51,857 INFO [M:0;51ca66f7ee3c:38429 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T20:55:51,857 DEBUG [M:0;51ca66f7ee3c:38429 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731358550538Disabling compacts and flushes for region at 1731358550538Disabling writes for close at 1731358550538Obtaining lock to block concurrent updates at 1731358550539 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731358550539Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=780559, getHeapSize=955968, getOffHeapSize=0, getCellsCount=2304 at 1731358550540 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731358550541 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731358550542 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731358550561 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731358550561Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731358550975 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731358550996 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731358550996Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731358551410 (+414 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731358551431 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731358551431Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5485d8b7: reopening flushed file at 1731358551844 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4ad10ec7: reopening flushed file at 1731358551849 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e8bda53: reopening flushed file at 1731358551852 (+3 ms)Finished flush of dataSize ~762.26 KB/780559, heapSize ~933.33 KB/955728, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1317ms, sequenceid=2125, compaction requested=false at 1731358551856 (+4 ms)Writing region close event to WAL at 1731358551857 (+1 ms)Closed at 1731358551857 2024-11-11T20:55:51,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34289 is added to blk_1073741830_1006 (size=919819) 2024-11-11T20:55:51,859 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/0171226a-3325-38ac-7fcc-7b386960db23/MasterData/WALs/51ca66f7ee3c,38429,1731358361462/51ca66f7ee3c%2C38429%2C1731358361462.1731358364276 not finished, retry = 0 2024-11-11T20:55:51,960 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T20:55:51,960 INFO [M:0;51ca66f7ee3c:38429 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-11T20:55:51,960 INFO [M:0;51ca66f7ee3c:38429 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38429 2024-11-11T20:55:51,961 INFO [M:0;51ca66f7ee3c:38429 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T20:55:52,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:55:52,062 INFO [M:0;51ca66f7ee3c:38429 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T20:55:52,062 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38429-0x100308f60b00000, quorum=127.0.0.1:61333, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T20:55:52,068 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T20:55:52,071 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T20:55:52,072 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T20:55:52,072 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T20:55:52,072 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/hadoop.log.dir/,STOPPED} 2024-11-11T20:55:52,074 WARN [BP-1400522365-172.17.0.2-1731358358860 heartbeating to localhost/127.0.0.1:45199 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T20:55:52,074 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T20:55:52,074 WARN [BP-1400522365-172.17.0.2-1731358358860 heartbeating to localhost/127.0.0.1:45199 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1400522365-172.17.0.2-1731358358860 (Datanode Uuid 4d7c653e-098e-43e9-9ddd-92046dcfd7fe) service to localhost/127.0.0.1:45199 2024-11-11T20:55:52,074 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T20:55:52,077 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/cluster_aeef4fcb-2790-f92a-34f3-fa13c2a932a9/data/data1/current/BP-1400522365-172.17.0.2-1731358358860 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T20:55:52,077 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/cluster_aeef4fcb-2790-f92a-34f3-fa13c2a932a9/data/data2/current/BP-1400522365-172.17.0.2-1731358358860 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T20:55:52,078 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T20:55:52,084 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T20:55:52,084 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T20:55:52,084 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T20:55:52,085 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T20:55:52,085 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/379a93e5-b5f7-62a0-3deb-51ae3618d925/hadoop.log.dir/,STOPPED} 2024-11-11T20:55:52,099 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-11T20:55:52,206 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down